3.60

Bumps [gocloud.dev/pubsub/rabbitpubsub](https://github.com/google/go-cloud) from 0.35.0 to 0.36.0.
- [Release notes](https://github.com/google/go-cloud/releases)
- [Commits](https://github.com/google/go-cloud/compare/v0.35.0...v0.36.0)

---
updated-dependencies:
- dependency-name: gocloud.dev/pubsub/rabbitpubsub
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/binaries_dev.yml

@@ -38,13 +38,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -87,13 +87,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release0.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release1.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release2.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release3.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release4.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -45,7 +45,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@0a0935d79e800ba5d9595ede793444fb1fdef368 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/codeql.yml

@@ -18,11 +18,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         # Override language selection by uncommenting this and choosing your languages
         with:
           languages: go
@@ -30,7 +30,7 @@ jobs:
       # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below).
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -44,4 +44,4 @@ jobs:
       #     make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/container_dev.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -56,7 +56,7 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -57,7 +57,7 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -48,7 +48,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -48,7 +48,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release5.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@9dc751fe249ad99385a2583ee0d084c400eee04e # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -47,7 +47,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/depsreview.yml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034
+        uses: actions/dependency-review-action@01bc87099ba56df1e897b6874784491ea6309bc4

.github/workflows/e2e.yml

@@ -24,13 +24,13 @@ jobs:
     timeout-minutes: 30
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v2
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
 
     - name: Install dependencies
       run: |
@@ -94,7 +94,7 @@ jobs:
 
     - name: Archive logs
       if: always()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: output-logs
         path: docker/output.log

.github/workflows/go.yml

@@ -21,20 +21,20 @@ jobs:
     steps:
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v2
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
 
     - name: Get dependencies
       run: |
         cd weed; go get -v -t -d ./...
 
     - name: Build
-      run: cd weed; go build -tags "elastic gocdk sqlite ydb tikv" -v .
+      run: cd weed; go build -tags "elastic gocdk sqlite ydb tikv rclone" -v .
 
     - name: Test
-      run: cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
+      run: cd weed; go test -tags "elastic gocdk sqlite ydb tikv rclone" -v ./...

.github/workflows/helm_chart_release.yml

@@ -1,8 +1,10 @@
 name: "helm: publish charts"
 on:
   push:
-    tags: 
-      - '*'
+    branches:
+      - master
+    paths:
+      - "k8s/charts/**"
 
 permissions:
   contents: write
@@ -12,7 +14,7 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       - name: Publish Helm charts
         uses: stefanprodan/helm-gh-pages@master
         with:          

.github/workflows/helm_ci.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
         with:
           fetch-depth: 0
 
@@ -25,27 +25,27 @@ jobs:
         with:
           version: v3.10.0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           check-latest: true
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.4.0
+        uses: helm/chart-testing-action@v2.6.1
 
       - name: Run chart-testing (list-changed)
         id: list-changed
         run: |
-          changed=$(ct list-changed --chart-dirs k8s/charts)
+          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} --chart-dirs k8s/charts)
           if [[ -n "$changed" ]]; then
             echo "::set-output name=changed::true"
           fi
 
       - name: Run chart-testing (lint)
-        run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts
+        run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
 
       - name: Create kind cluster
         uses: helm/kind-action@v1.8.0
 
       - name: Run chart-testing (install)
-        run: ct install --all --chart-dirs k8s/charts
+        run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts

.gitignore

@@ -88,3 +88,7 @@ other/java/hdfs/dependency-reduced-pom.xml
 # binary file
 weed/weed
 weed/mq/client/cmd/weed_pub/weed_pub
+docker/weed
+
+# test generated files
+weed/*/*.jpg

Makefile

@@ -1,14 +1,35 @@
 BINARY = weed
 
 SOURCE_DIR = .
+debug ?= 0
 
 all: install
 
 install:
 	cd weed; go install
 
-full_install:
-	cd weed; go install -tags "elastic gocdk sqlite ydb tikv"
+warp_install:
+	go install github.com/minio/warp@v0.7.6
 
-tests:
-	cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
+full_install:
+	cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone"
+
+server: install
+	weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json
+
+benchmark: install warp_install
+	pkill weed || true
+	pkill warp || true
+	weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
+	warp client &
+	while ! nc -z localhost 8000 ; do sleep 1 ; done
+	warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm
+	pkill warp
+	pkill weed
+
+# curl -o profile "http://127.0.0.1:6060/debug/pprof/profile?debug=1"
+benchmark_with_pprof: debug = 1
+benchmark_with_pprof: benchmark
+
+test:
+	cd weed; go test -tags "elastic gocdk sqlite ydb tikv rclone" -v ./...

README.md

@@ -582,6 +582,71 @@ Percentage of the requests served within a certain time (ms)
   100%     54.1 ms
 ```
 
+### Run WARP and launch a mixed benchmark. ###
+
+```
+make benchmark
+warp: Benchmark data written to "warp-mixed-2023-10-16[102354]-l70a.csv.zst"                                                                                                                                                                                               
+Mixed operations.
+Operation: DELETE, 10%, Concurrency: 20, Ran 4m59s.
+ * Throughput: 6.19 obj/s
+
+Operation: GET, 45%, Concurrency: 20, Ran 5m0s.
+ * Throughput: 279.85 MiB/s, 27.99 obj/s
+
+Operation: PUT, 15%, Concurrency: 20, Ran 5m0s.
+ * Throughput: 89.86 MiB/s, 8.99 obj/s
+
+Operation: STAT, 30%, Concurrency: 20, Ran 5m0s.
+ * Throughput: 18.63 obj/s
+
+Cluster Total: 369.74 MiB/s, 61.79 obj/s, 0 errors over 5m0s.
+```
+
+To see segmented request statistics, use the --analyze.v parameter.
+```
+warp analyze --analyze.v warp-mixed-2023-10-16[102354]-l70a.csv.zst
+18642 operations loaded... Done!
+Mixed operations.
+----------------------------------------
+Operation: DELETE - total: 1854, 10.0%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
+ * Throughput: 6.19 obj/s
+
+Requests considered: 1855:
+ * Avg: 104ms, 50%: 30ms, 90%: 207ms, 99%: 1.355s, Fastest: 1ms, Slowest: 4.613s, StdDev: 320ms
+
+----------------------------------------
+Operation: GET - total: 8388, 45.3%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.12 +0500 +05
+ * Throughput: 279.77 MiB/s, 27.98 obj/s
+
+Requests considered: 8389:
+ * Avg: 221ms, 50%: 106ms, 90%: 492ms, 99%: 1.739s, Fastest: 8ms, Slowest: 8.633s, StdDev: 383ms
+ * TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 171ms, 99th: 669ms, Worst: 4.783s StdDev: 163ms
+ * First Access: Avg: 240ms, 50%: 105ms, 90%: 511ms, 99%: 2.08s, Fastest: 12ms, Slowest: 8.633s, StdDev: 480ms
+ * First Access TTFB: Avg: 88ms, Best: 2ms, 25th: 24ms, Median: 38ms, 75th: 64ms, 90th: 179ms, 99th: 919ms, Worst: 4.783s StdDev: 199ms
+ * Last Access: Avg: 219ms, 50%: 106ms, 90%: 463ms, 99%: 1.782s, Fastest: 9ms, Slowest: 8.633s, StdDev: 416ms
+ * Last Access TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 161ms, 99th: 657ms, Worst: 4.783s StdDev: 176ms
+
+----------------------------------------
+Operation: PUT - total: 2688, 14.5%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
+ * Throughput: 89.83 MiB/s, 8.98 obj/s
+
+Requests considered: 2689:
+ * Avg: 1.165s, 50%: 878ms, 90%: 2.015s, 99%: 5.74s, Fastest: 99ms, Slowest: 8.264s, StdDev: 968ms
+
+----------------------------------------
+Operation: STAT - total: 5586, 30.2%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.113 +0500 +05
+ * Throughput: 18.63 obj/s
+
+Requests considered: 5587:
+ * Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 80ms, Fastest: 0s, Slowest: 245ms, StdDev: 17ms
+ * First Access: Avg: 14ms, 50%: 10ms, 90%: 33ms, 99%: 69ms, Fastest: 0s, Slowest: 203ms, StdDev: 16ms
+ * Last Access: Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 74ms, Fastest: 0s, Slowest: 203ms, StdDev: 17ms
+
+Cluster Total: 369.64 MiB/s, 61.77 obj/s, 0 errors over 5m0s.
+Total Errors:0.
+```
+
 [Back to TOC](#table-of-contents)
 
 ## License ##

docker/Dockerfile.gccgo_build

@@ -1,44 +0,0 @@
-FROM gcc:11 as builder
-RUN mkdir -p /go/src/github.com/seaweedfs/
-RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
-ARG BRANCH=${BRANCH:-master}
-RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
-RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
-  && apt-get update \
-  && apt-get install -y golang-src \
-  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
-  && CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm
-
-FROM alpine AS final
-LABEL author="Chris Lu"
-COPY --from=builder /go/bin/weed /usr/bin/
-RUN mkdir -p /etc/seaweedfs
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
-RUN apk add fuse # for weed mount
-
-# volume server gprc port
-EXPOSE 18080
-# volume server http port
-EXPOSE 8080
-# filer server gprc port
-EXPOSE 18888
-# filer server http port
-EXPOSE 8888
-# master server shared gprc port
-EXPOSE 19333
-# master server shared http port
-EXPOSE 9333
-# s3 server http port
-EXPOSE 8333
-# webdav server http port
-EXPOSE 7333
-
-RUN mkdir -p /data/filerldb2
-
-VOLUME /data
-WORKDIR /data
-
-RUN chmod +x /entrypoint.sh
-
-ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.go_build

@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine as builder
+FROM golang:1.21-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/seaweedfs/
 RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs

docker/Dockerfile.rocksdb_dev_env

@@ -1,4 +1,4 @@
-FROM golang:1.20-buster as builder
+FROM golang:1.21-buster as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev

docker/Dockerfile.rocksdb_large

@@ -1,4 +1,4 @@
-FROM golang:1.20-buster as builder
+FROM golang:1.21-buster as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev

docker/Makefile

@@ -102,12 +102,12 @@ clean:
 	rm ./weed
 
 certstrap:
-	go get github.com/square/certstrap
-	certstrap --depot-path compose/tls init --passphrase "" --common-name "SeaweedFS CA" || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name volume01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name master01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name filer01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name client01.dev || true
+	go install -v github.com/square/certstrap@latest
+	certstrap --depot-path compose/tls init --curve P-256 --passphrase "" --common-name "SeaweedFS CA" || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --domain localhost --common-name volume01.dev || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name master01.dev || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name filer01.dev || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name client01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true

docker/compose/local-mq-test.yml

@@ -0,0 +1,27 @@
+version: '3.9'
+
+services:
+  server:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+      - 8888:8888
+      - 18888:18888
+    command: "server -ip=server -filer -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1"
+    healthcheck:
+      test: curl -f http://localhost:8888/healthz
+  mq_broker:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 17777:17777
+    command: "mq.broker -master=server:9333 -ip=mq_broker"
+    depends_on:
+      server:
+        condition: service_healthy
+  mq_client:
+    image: chrislusf/seaweedfs:local
+    # run a custom command instead of entrypoint
+    command: "ls -al"
+    depends_on:
+      - mq_broker

docker/compose/s3.json

@@ -40,7 +40,10 @@
         "List",
         "Tagging",
         "Write"
-      ]
+      ],
+      "account": {
+        "id": "testid"
+      }
     },
     {
       "name": "s3_tests_alt",
@@ -101,5 +104,12 @@
         "Write"
       ]
     }
-  ]
+  ],
+  "accounts": [
+      {
+        "id" : "testid",
+        "displayName": "M. Tester",
+        "emailAddress": "tester@ceph.com"
+      }
+    ]
 }

docker/compose/s3tests.conf

@@ -18,10 +18,10 @@ bucket prefix = yournamehere-{random}-
 
 [s3 main]
 # main display_name set in vstart.sh
-display_name = s3_tests
+display_name = M. Tester
 
 # main user_idname set in vstart.sh
-user_id = s3_tests
+user_id = testid
 
 # main email set in vstart.sh
 email = tester@ceph.com

docker/compose/tls.env

@@ -12,5 +12,9 @@ WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,clie
 WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
 WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
 WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
+WEED_HTTPS_CLIENT_ENABLE=true
+WEED_HTTPS_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt
+WEED_HTTPS_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key
+WEED_HTTPS_VOLUME_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt
 #GRPC_GO_LOG_SEVERITY_LEVEL=info
 #GRPC_GO_LOG_VERBOSITY_LEVEL=2

go.mod

@@ -1,22 +1,22 @@
 module github.com/seaweedfs/seaweedfs
 
-go 1.20
+go 1.21
 
 require (
-	cloud.google.com/go v0.110.7 // indirect
+	cloud.google.com/go v0.110.10 // indirect
 	cloud.google.com/go/pubsub v1.33.0
-	cloud.google.com/go/storage v1.32.0
+	cloud.google.com/go/storage v1.36.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.38.1
-	github.com/aws/aws-sdk-go v1.44.314
+	github.com/aws/aws-sdk-go v1.49.0
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.1
@@ -29,10 +29,10 @@ require (
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fclairamb/ftpserverlib v0.22.0
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/go-redsync/redsync/v4 v4.8.2
+	github.com/go-redsync/redsync/v4 v4.11.0
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.6.0
@@ -41,11 +41,10 @@ require (
 	github.com/golang/protobuf v1.5.3
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/uuid v1.3.1
+	github.com/google/uuid v1.5.0
 	github.com/google/wire v0.5.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
-	github.com/gorilla/mux v1.8.0
+	github.com/gorilla/mux v1.8.1
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -58,42 +57,40 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
+	github.com/klauspost/compress v1.17.3 // indirect
 	github.com/klauspost/reedsolomon v1.11.8
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.8.0
+	github.com/linxGnu/grocksdb v1.8.10
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/peterh/liner v1.2.2
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.2.0
-	github.com/prometheus/client_golang v1.16.0
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.11.1
+	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
 	github.com/seaweedfs/raft v1.1.3
-	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/spf13/afero v1.9.5 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/spf13/afero v1.10.0 // indirect
 	github.com/spf13/cast v1.5.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.16.0
+	github.com/spf13/viper v1.17.0
 	github.com/stretchr/testify v1.8.4
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tidwall/gjson v1.16.0
+	github.com/tidwall/gjson v1.17.0
 	github.com/tidwall/match v1.1.1
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
@@ -106,215 +103,242 @@ require (
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	go.etcd.io/etcd/client/v3 v3.5.9
-	go.mongodb.org/mongo-driver v1.12.1
+	go.etcd.io/etcd/client/v3 v3.5.11
+	go.mongodb.org/mongo-driver v1.13.1
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.34.0
-	gocloud.dev/pubsub/natspubsub v0.33.0
-	gocloud.dev/pubsub/rabbitpubsub v0.33.0
-	golang.org/x/crypto v0.13.0 // indirect
-	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
-	golang.org/x/image v0.11.0
-	golang.org/x/net v0.15.0
-	golang.org/x/oauth2 v0.12.0 // indirect
-	golang.org/x/sys v0.12.0
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/tools v0.11.0
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.141.0
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
-	google.golang.org/grpc v1.58.0
-	google.golang.org/protobuf v1.31.0
+	gocloud.dev v0.36.0
+	gocloud.dev/pubsub/natspubsub v0.35.0
+	gocloud.dev/pubsub/rabbitpubsub v0.36.0
+	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/image v0.14.0
+	golang.org/x/net v0.19.0
+	golang.org/x/oauth2 v0.14.0 // indirect
+	golang.org/x/sys v0.15.0
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.16.1
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/api v0.151.0
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect
+	google.golang.org/grpc v1.59.0
+	google.golang.org/protobuf v1.32.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect
 	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.24.1 // indirect
-	modernc.org/mathutil v1.6.0 // indirect
-	modernc.org/memory v1.6.0 // indirect
+	modernc.org/libc v1.29.0 // indirect
+	modernc.org/mathutil v1.6.0
+	modernc.org/memory v1.7.2 // indirect
 	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sqlite v1.25.0
-	modernc.org/strutil v1.1.3
+	modernc.org/sqlite v1.27.0
+	modernc.org/strutil v1.2.0
 	modernc.org/token v1.0.1 // indirect
 )
 
 require (
-	github.com/Jille/raft-grpc-transport v1.4.0
-	github.com/arangodb/go-driver v1.6.0
+	github.com/Jille/raft-grpc-transport v1.5.0
+	github.com/arangodb/go-driver v1.6.1
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.21.0
-	github.com/aws/aws-sdk-go-v2/config v1.18.39
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.37
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
+	github.com/aws/aws-sdk-go-v2 v1.24.0
+	github.com/aws/aws-sdk-go-v2/config v1.26.2
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.13
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.47.5
 	github.com/fluent/fluent-logger-golang v1.9.0
-	github.com/golang-jwt/jwt/v5 v5.0.0
+	github.com/golang-jwt/jwt/v5 v5.1.0
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
 	github.com/hanwen/go-fuse/v2 v2.4.0
-	github.com/hashicorp/raft v1.5.0
-	github.com/hashicorp/raft-boltdb/v2 v2.2.2
+	github.com/hashicorp/raft v1.6.0
+	github.com/hashicorp/raft-boltdb/v2 v2.3.0
 	github.com/orcaman/concurrent-map/v2 v2.0.1
-	github.com/puzpuzpuz/xsync/v2 v2.5.0
-	github.com/rabbitmq/amqp091-go v1.8.1
-	github.com/rclone/rclone v1.63.1
-	github.com/rdleal/intervalst v0.0.0-20221028215511-a098aa0d2cb8
-	github.com/schollz/progressbar/v3 v3.13.1
-	github.com/shirou/gopsutil/v3 v3.23.5
+	github.com/puzpuzpuz/xsync/v2 v2.5.1
+	github.com/rabbitmq/amqp091-go v1.9.0
+	github.com/rclone/rclone v1.65.0
+	github.com/rdleal/intervalst v1.2.1
+	github.com/schollz/progressbar/v3 v3.14.1
+	github.com/shirou/gopsutil/v3 v3.23.11
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.48.8
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
 
 require (
-	cloud.google.com/go/compute v1.23.0 // indirect
+	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 // indirect
+	cloud.google.com/go/iam v1.1.5 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
-	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
+	github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect
+	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
+	github.com/ProtonMail/go-srp v0.0.7 // indirect
+	github.com/ProtonMail/gopenpgp/v2 v2.7.4 // indirect
+	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/Unknwon/goconfig v1.0.0 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect
-	github.com/aws/smithy-go v1.14.2 // indirect
-	github.com/benbjohnson/clock v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.9 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.26.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.6 // indirect
+	github.com/aws/smithy-go v1.19.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/bradenaw/juniper v0.13.1 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
-	github.com/colinmarc/hdfs/v2 v2.3.0 // indirect
+	github.com/cloudflare/circl v1.3.6 // indirect
+	github.com/cloudsoda/go-smb2 v0.0.0-20231106205947-b0758ecc4c67 // indirect
+	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
+	github.com/cronokirby/saferith v0.33.0 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
+	github.com/emersion/go-message v0.17.0 // indirect
+	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
+	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
 	github.com/fatih/color v1.14.1 // indirect
 	github.com/fclairamb/go-log v0.4.1 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/flynn/noise v1.0.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
-	github.com/go-chi/chi/v5 v5.0.8 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-chi/chi/v5 v5.0.10 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-resty/resty/v2 v2.10.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-msgpack v1.1.5 // indirect
+	github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
-	github.com/hirochachacha/go-smb2 v1.1.0 // indirect
+	github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
+	github.com/henrybear327/go-proton-api v1.0.0 // indirect
 	github.com/iguanesolutions/go-systemd/v5 v5.1.1 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/jlaffaye/ftp v0.2.0 // indirect
 	github.com/jonboulle/clockwork v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
+	github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect
+	github.com/jtolio/noiseconn v0.0.0-20230621152802-afeab29449e0 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.7.0 // indirect
-	github.com/nats-io/nats.go v1.28.0 // indirect
-	github.com/nats-io/nkeys v0.4.4 // indirect
+	github.com/nats-io/nats.go v1.31.0 // indirect
+	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
-	github.com/ncw/swift/v2 v2.0.1 // indirect
+	github.com/ncw/swift/v2 v2.0.2 // indirect
+	github.com/onsi/gomega v1.27.10 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.34.0 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.51.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
-	github.com/philhofer/fwd v1.1.1 // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
 	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
-	github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 // indirect
+	github.com/pkg/sftp v1.13.6 // indirect
 	github.com/pkg/xattr v0.4.9 // indirect
-	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
-	github.com/rclone/ftp v0.0.0-20230327202000-dadc1f64e87d // indirect
-	github.com/rdleal/intervalst v0.0.0-20221028215511-a098aa0d2cb8 // indirect
+	github.com/relvacode/iso8601 v1.3.0 // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/sagikazarmark/locafero v0.3.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
 	github.com/sony/gobreaker v0.5.0 // indirect
-	github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca // indirect
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
-	github.com/tinylib/msgp v1.1.6 // indirect
-	github.com/tklauser/go-sysconf v0.3.11 // indirect
-	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222 // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20231012155159-f85a672542fd // indirect
 	github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
-	go.etcd.io/bbolt v1.3.7 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
+	go.etcd.io/bbolt v1.3.8 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.11 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.11 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.12.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/time v0.4.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect
-	storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
-	storj.io/drpc v0.0.32 // indirect
-	storj.io/uplink v1.10.0 // indirect
+	storj.io/common v0.0.0-20231027080355-b4cb1b0d728e // indirect
+	storj.io/drpc v0.0.33 // indirect
+	storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
+	storj.io/uplink v1.12.1 // indirect
 )
 
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft

k8s/charts/seaweedfs/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.57"
-version: 3.57.0
+appVersion: "3.60"
+version: 3.60.1

k8s/charts/seaweedfs/README.md

@@ -4,15 +4,21 @@
 
 ### Add the helm repo
 
-`helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm`
+```bash
+helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm
+```
 
 ### Install the helm chart
 
-`helm install seaweedfs seaweedfs/seaweedfs`
+```bash
+helm install seaweedfs seaweedfs/seaweedfs
+```
 
 ### (Recommended) Provide `values.yaml`
 
-`helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs`
+```bash
+helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs
+```
 
 ## Info:
 * master/filer/volume are stateful sets with anti-affinity on the hostname,
@@ -78,4 +84,63 @@ for your deployment.
 You can update the replicas count for each node type in values.yaml,
 need to add more nodes with the corresponding labels if applicable.
 
-Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
+Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
+
+## S3 configuration
+
+To enable an s3 endpoint for your filer with a default install add the following to your values.yaml:
+
+```yaml
+filer:
+  s3:
+    enabled: true
+```
+
+### Enabling Authenticaion to S3
+
+To enable authentication for S3, you have two options:
+
+- let the helm chart create an admin user as well as a read only user
+- provide your own s3 config.json file via an existing Kubernetes Secret
+
+#### Use the default credentials for S3
+
+Example parameters for your values.yaml:
+
+```yaml
+filer:
+  s3:
+    enabled: true
+    enableAuth: true
+```
+
+#### Provide your own credentials for S3
+
+Example parameters for your values.yaml:
+
+```yaml
+filer:
+  s3:
+    enabled: true
+    enableAuth: true
+    existingConfigSecret: my-s3-secret
+```
+
+Example existing secret with your s3 config to create an admin user and readonly user, both with credentials:
+
+```yaml
+---
+# Source: seaweedfs/templates/seaweedfs-s3-secret.yaml
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: my-s3-secret
+  namespace: seaweedfs
+  labels:
+    app.kubernetes.io/name: seaweedfs
+    app.kubernetes.io/component: s3
+stringData:
+  # this key must be an inline json config file
+  seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"snu8yoP6QAlY0ne4","secretKey":"PNzBcmeLNEdR0oviwm04NQAicOrDH1Km"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"SCigFee6c5lbi04A","secretKey":"kgFhbT38R8WUYVtiFQ1OiSVOrYr3NKku"}],"actions":["Read"]}]}'
+```

k8s/charts/seaweedfs/templates/_helpers.tpl

@@ -118,20 +118,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-
-{{/* check if any Volume HostPath exists */}}
-{{- define "volume.hostpath_exists" -}}
-{{- if or (or (eq .Values.volume.data.type "hostPath") (and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "hostPath") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- if or .Values.global.enableSecurity .Values.volume.extraVolumes -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
-{{- end -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
@@ -140,16 +127,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- if or (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-
-{{/* check if any Filer HostPath exists */}}
-{{- define "filer.hostpath_exists" -}}
-{{- if or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.logs.type "hostPath") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
@@ -158,20 +136,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- if or (eq .Values.master.data.type "persistentVolumeClaim") (eq .Values.master.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-
-{{/* check if any Master HostPath exists */}}
-{{- define "master.hostpath_exists" -}}
-{{- if or (eq .Values.master.data.type "hostPath") (eq .Values.master.logs.type "hostPath") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- if or .Values.global.enableSecurity .Values.volume.extraVolumes -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
-{{- end -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
@@ -180,7 +145,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- if or (not (empty .Values.volume.dir_idx )) (not (empty .Values.volume.initContainers )) -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
@@ -198,3 +163,22 @@ imagePullSecrets:
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{/*
+Renders a value that contains template perhaps with scope if the scope is present.
+Usage:
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
+*/}}
+{{- define "common.tplvalues.render" -}}
+{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
+{{- if contains "{{" (toJson .value) }}
+  {{- if .scope }}
+      {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }}
+  {{- else }}
+    {{- tpl $value .context }}
+  {{- end }}
+{{- else }}
+    {{- $value }}
+{{- end }}
+{{- end -}}

k8s/charts/seaweedfs/templates/ca-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/cert-caissuer.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Issuer
 metadata:

k8s/charts/seaweedfs/templates/cert-clusterissuer.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: ClusterIssuer
 metadata:

k8s/charts/seaweedfs/templates/client-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/filer-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -12,7 +12,7 @@ metadata:
     app.kubernetes.io/component: filer
 spec:
   serviceName: {{ template "seaweedfs.name" . }}-filer
-  podManagementPolicy: Parallel
+  podManagementPolicy: {{ .Values.filer.podManagementPolicy }}
   replicas: {{ .Values.filer.replicas }}
   {{- if (gt (int .Values.filer.updatePartition) 0) }}
   updateStrategy:
@@ -33,6 +33,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: filer
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.filer.restartPolicy }}
       {{- if .Values.filer.affinity }}
@@ -45,16 +52,13 @@ spec:
       {{- end }}
       {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
       {{- if .Values.global.createClusterRole }}
-      serviceAccountName: seaweedfs-rw-sa #hack for delete pod master after migration
+      serviceAccountName: {{ .Values.filer.serviceAccountName | default .Values.global.serviceAccountName | quote }} # for deleting statefulset pods after migration
       {{- end }}
       terminationGracePeriodSeconds: 60
       {{- if .Values.filer.priorityClassName }}
       priorityClassName: {{ .Values.filer.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if .Values.filer.serviceAccountName }}
-      serviceAccountName: {{ .Values.filer.serviceAccountName | quote }}
-      {{- end }}
       {{- if .Values.filer.initContainers }}
       initContainers:
         {{ tpl .Values.filer.initContainers . | nindent 8 | trim }}
@@ -110,11 +114,11 @@ spec:
             - name: {{ $key }}
               valueFrom: {{ toYaml $value | nindent 16 }}
             {{- end }}
-            {{- end }}             
+            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
-            - | 
+            - |
               exec /usr/bin/weed \
               {{- if eq .Values.filer.logs.type "hostPath" }}
               -logdir=/logs \
@@ -188,9 +192,11 @@ spec:
             - name: seaweedfs-filer-log-volume
               mountPath: "/logs/"
             {{- end }}
-            - mountPath: /etc/sw
-              name: config-users
+            {{- if .Values.filer.s3.enableAuth }}
+            - name: config-users
+              mountPath: /etc/sw
               readOnly: true
+            {{- end }}
             {{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim"))) }}
             - name: data-filer
               mountPath: /data
@@ -252,6 +258,9 @@ spec:
           resources:
             {{ tpl .Values.filer.resources . | nindent 12 | trim }}
           {{- end }}
+        {{- if .Values.filer.sidecars }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
+        {{- end }}
       volumes:
         {{- if eq .Values.filer.logs.type "hostPath" }}
         - name: seaweedfs-filer-log-volume
@@ -259,19 +268,35 @@ spec:
             path: {{ .Values.filer.logs.hostPathPrefix }}/logs/seaweedfs/filer
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.filer.logs.type "existingClaim" }}
+        - name: seaweedfs-filer-log-volume
+          persistentVolumeClaim:
+            claimName: {{ .Values.filer.logs.claimName }}
+        {{- end }}
         {{- if eq .Values.filer.data.type "hostPath" }}
         - name: data-filer
           hostPath:
             path: {{ .Values.filer.data.hostPathPrefix }}/filer_store
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.filer.data.type "existingClaim" }}
+        - name: data-filer
+          persistentVolumeClaim:
+            claimName: {{ .Values.filer.data.claimName }}
+        {{- end }}
         - name: db-schema-config-volume
           configMap:
             name: seaweedfs-db-init-config
+        {{- if .Values.filer.s3.enableAuth }}
         - name: config-users
           secret:
             defaultMode: 420
+            {{- if .Values.filer.s3.existingConfigSecret }}
+            secretName: {{ .Values.filer.s3.existingConfigSecret }}
+            {{- else }}
             secretName: seaweedfs-s3-secret
+            {{- end }}
+        {{- end }}
         {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:
@@ -297,7 +322,7 @@ spec:
       nodeSelector:
         {{ tpl .Values.filer.nodeSelector . | indent 8 | trim }}
       {{- end }}
-  {{- if .Values.filer.enablePVC }}
+  {{- if and (.Values.filer.enablePVC) (eq .Values.filer.data.type "persistentVolumeClaim") }}
   # DEPRECATION: Deprecate in favor of filer.data section below
   volumeClaimTemplates:
   - metadata:
@@ -315,9 +340,13 @@ spec:
   {{- $pvc_exists := include "filer.pvc_exists" . -}}
   {{- if $pvc_exists }}
   volumeClaimTemplates:
-    {{- if eq .Values.filer.data.type "persistentVolumeClaim"}}
+    {{- if eq .Values.filer.data.type "persistentVolumeClaim" }}
     - metadata:
         name: data-filer
+        {{- with .Values.filer.data.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.filer.data.storageClass }}
@@ -325,9 +354,13 @@ spec:
           requests:
             storage: {{ .Values.filer.data.size }}
     {{- end }}
-    {{- if eq .Values.filer.logs.type "persistentVolumeClaim"}}
+    {{- if eq .Values.filer.logs.type "persistentVolumeClaim" }}
     - metadata:
         name: seaweedfs-filer-log-volume
+        {{- with .Values.filer.logs.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.filer.logs.storageClass }}

k8s/charts/seaweedfs/templates/master-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/master-ingress.yaml

@@ -22,6 +22,8 @@ metadata:
     app.kubernetes.io/component: master
 spec:
   ingressClassName: {{ .Values.master.ingress.className | quote }}
+  tls:
+    {{ .Values.master.ingress.tls | default list | toYaml | nindent 6 }}
   rules:
     - http:
         paths:

k8s/charts/seaweedfs/templates/master-servicemonitor.yaml

@@ -18,7 +18,7 @@ spec:
       scrapeTimeout: 5s
   selector:
     matchLabels:
-      app: {{ template "seaweedfs.name" . }}
-      component: master
+      app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+      app.kubernetes.io/component: master
 {{- end }}
 {{- end }}

k8s/charts/seaweedfs/templates/master-statefulset.yaml

@@ -11,7 +11,7 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
 spec:
   serviceName: {{ template "seaweedfs.name" . }}-master
-  podManagementPolicy: Parallel
+  podManagementPolicy: {{ .Values.master.podManagementPolicy }}
   replicas: {{ .Values.master.replicas }}
   {{- if (gt (int .Values.master.updatePartition) 0) }}
   updateStrategy:
@@ -32,6 +32,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: master
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.master.restartPolicy }}
       {{- if .Values.master.affinity }}
@@ -48,8 +55,8 @@ spec:
       priorityClassName: {{ .Values.master.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if .Values.master.serviceAccountName }}
-      serviceAccountName: {{ .Values.master.serviceAccountName | quote }}
+      {{- if .Values.global.createClusterRole }}
+      serviceAccountName: {{ .Values.master.serviceAccountName | default .Values.global.serviceAccountName | quote }} # for deleting statefulset pods after migration
       {{- end }}
       {{- if .Values.master.initContainers }}
       initContainers:
@@ -119,10 +126,11 @@ spec:
               {{- if .Values.master.metricsIntervalSec }}
               -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
               {{- end }}
-              {{- else if .Values.master.metricsPort }}
-              -metricsPort={{ .Values.master.metricsPort }} \
               {{- end }}
               {{- end }}
+              {{- if .Values.master.metricsPort }}
+              -metricsPort={{ .Values.master.metricsPort }} \
+              {{- end }}
               -volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \
               {{- if .Values.master.disableHttp }}
               -disableHttp \
@@ -205,8 +213,9 @@ spec:
           resources:
             {{ tpl .Values.master.resources . | nindent 12 | trim }}
           {{- end }}
-      {{- $hostpath_exists := include "master.hostpath_exists" . -}}
-      {{- if $hostpath_exists }}
+        {{- if .Values.master.sidecars }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
+        {{- end }}
       volumes:
         {{- if eq .Values.master.logs.type "hostPath" }}
         - name: seaweedfs-master-log-volume
@@ -214,12 +223,22 @@ spec:
             path: {{ .Values.master.logs.hostPathPrefix }}/logs/seaweedfs/master
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.master.logs.type "existingClaim" }}
+        - name: seaweedfs-master-log-volume
+          persistentVolumeClaim:
+            claimName: {{ .Values.master.logs.claimName }} 
+        {{- end }}
         {{- if eq .Values.master.data.type "hostPath" }}
         - name: data-{{ .Release.Namespace }}
           hostPath:
             path: {{ .Values.master.data.hostPathPrefix }}/seaweed-master/
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.master.data.type "existingClaim" }}
+        - name: data-{{ .Release.Namespace }}
+          persistentVolumeClaim:
+            claimName: {{ .Values.master.data.claimName }} 
+        {{- end }}
         - name: master-config
           configMap:
             name: {{ template "seaweedfs.name" . }}-master-config
@@ -244,7 +263,6 @@ spec:
             secretName: {{ template "seaweedfs.name" . }}-client-cert
         {{- end }}
         {{ tpl .Values.master.extraVolumes . | indent 8 | trim }}
-      {{- end }}
       {{- if .Values.master.nodeSelector }}
       nodeSelector:
         {{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
@@ -255,6 +273,10 @@ spec:
     {{- if eq .Values.master.data.type "persistentVolumeClaim"}}
     - metadata:
         name: data-{{ .Release.Namespace }}
+        {{- with .Values.master.data.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.master.data.storageClass }}
@@ -265,6 +287,10 @@ spec:
     {{- if eq .Values.master.logs.type "persistentVolumeClaim"}}
     - metadata:
         name: seaweedfs-master-log-volume
+        {{- with .Values.master.logs.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.master.logs.storageClass }}

k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml

@@ -0,0 +1,96 @@
+{{- if .Values.master.enabled }}
+{{- if .Values.filer.s3.enabled }}
+{{- if .Values.filer.s3.createBuckets }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ $.Release.Name }}-bucket-hook"
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+      labels:
+        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+        app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: post-install-job
+        image: {{ template "master.image" . }}
+        env:
+          - name: WEED_CLUSTER_DEFAULT
+            value: "sw"
+          - name: WEED_CLUSTER_SW_MASTER
+            value: "{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}:9333"
+          - name: WEED_CLUSTER_SW_FILER
+            value: "{{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:8888"
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: SEAWEEDFS_FULLNAME
+            value: "{{ template "seaweedfs.name" . }}"
+        command:
+          - "/bin/sh"
+          - "-ec"
+          - |
+          {{- range $reg, $props := $.Values.filer.s3.createBuckets }}
+            exec /bin/echo \
+            "s3.bucket.create --name {{ $props.name }}" |\
+            /usr/bin/weed shell
+          {{- end }}
+          {{- range $reg, $props := $.Values.filer.s3.createBuckets }}
+          {{- if $props.anonymousRead }}
+            exec /bin/echo \
+            "s3.configure --user anonymous \
+                --buckets {{ $props.name }} \
+                --actions Read \
+                --apply true" |\
+                /usr/bin/weed shell
+          {{- end }}  
+          {{- end }}
+        {{- if .Values.filer.s3.enableAuth }}
+        volumeMounts:
+          - name: config-users
+            mountPath: /etc/sw
+            readOnly: true
+        {{- end }}
+        ports:
+          - containerPort: {{ .Values.master.port }}
+            name: swfs-master
+          {{- if and .Values.global.monitoring.enabled .Values.master.metricsPort }}
+          - containerPort: {{ .Values.master.metricsPort }}
+            name: metrics
+          {{- end }}
+          - containerPort: {{ .Values.master.grpcPort }}
+            #name: swfs-master-grpc
+    {{- if .Values.filer.s3.enableAuth }}
+      volumes:
+        - name: config-users
+          secret:
+            defaultMode: 420
+            {{- if not (empty .Values.filer.s3.existingConfigSecret) }}
+            secretName: {{ .Values.filer.s3.existingConfigSecret }}
+            {{- else }}
+            secretName: seaweedfs-s3-secret
+            {{- end }}
+    {{- end }}{{/** if .Values.filer.s3.enableAuth **/}}
+{{- end }}{{/** if .Values.master.enabled **/}}
+{{- end }}{{/** if .Values.filer.s3.enabled **/}}
+{{- end }}{{/** if .Values.filer.s3.createBuckets **/}}

k8s/charts/seaweedfs/templates/s3-deployment.yaml

@@ -24,6 +24,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: s3
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.s3.restartPolicy }}
       {{- if .Values.s3.tolerations }}
@@ -71,7 +78,7 @@ spec:
           command:
             - "/bin/sh"
             - "-ec"
-            - | 
+            - |
               exec /usr/bin/weed \
                {{- if eq .Values.s3.logs.type "hostPath" }}
               -logdir=/logs \
@@ -114,9 +121,11 @@ spec:
             - name: logs
               mountPath: "/logs/"
             {{- end }}
+            {{- if .Values.s3.enableAuth }}
             - mountPath: /etc/sw
               name: config-users
               readOnly: true
+            {{- end }}
             {{- if .Values.global.enableSecurity }}
             - name: security-config
               readOnly: true
@@ -174,11 +183,20 @@ spec:
           resources:
             {{ tpl .Values.s3.resources . | nindent 12 | trim }}
           {{- end }}
+      {{- if .Values.s3.sidecars }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
+      {{- end }}
       volumes:
+        {{- if .Values.s3.enableAuth }}
         - name: config-users
           secret:
             defaultMode: 420
+            {{- if .Values.filer.s3.existingConfigSecret }}
+            secretName: {{ .Values.s3.existingConfigSecret }}
+            {{- else }}
             secretName: seaweedfs-s3-secret
+            {{- end }}
+        {{- end }}
         {{- if eq .Values.s3.logs.type "hostPath" }}
         - name: logs
           hostPath:

k8s/charts/seaweedfs/templates/s3-ingress.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.s3.ingress.enabled }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1beta1
+{{- else }}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: ingress-{{ template "seaweedfs.name" . }}-s3
+  namespace: {{ .Release.Namespace }}
+  {{- with .Values.s3.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: s3
+spec:
+  ingressClassName: {{ .Values.s3.ingress.className | quote }}
+  tls:
+    {{ .Values.s3.ingress.tls | default list | toYaml | nindent 6}}
+  rules:
+  - http:
+      paths:
+      - path: /
+        pathType: ImplementationSpecific
+        backend:
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ template "seaweedfs.name" . }}-s3
+            port:
+              number: {{ .Values.s3.port }}
+              #name:
+{{- else }}
+          serviceName: {{ template "seaweedfs.name" . }}-s3
+          servicePort: {{ .Values.s3.port }}
+{{- end }}
+{{- if .Values.s3.ingress.host }}
+    host: {{ .Values.s3.ingress.host }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/s3-secret.yaml

@@ -1,4 +1,4 @@
-{{- if not (or .Values.filer.s3.skipAuthSecretCreation .Values.s3.skipAuthSecretCreation) }}
+{{- if not (or .Values.filer.s3.skipAuthSecretCreation .Values.s3.skipAuthSecretCreation .Values.filer.s3.existingConfigSecret .Values.s3.existingConfigSecret ) }}
 {{- $access_key_admin := randAlphaNum 16 -}}
 {{- $secret_key_admin := randAlphaNum 32 -}}
 {{- $access_key_read := randAlphaNum 16 -}}
@@ -32,4 +32,4 @@ stringData:
   s3_auditLogConfig.json: |
     {{ toJson .Values.s3.auditLogConfig | nindent 4 }}
   {{- end }}
-{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/s3-service.yaml

@@ -9,6 +9,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
+  internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }}
   ports:
   - name: "swfs-s3"
     port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}

k8s/charts/seaweedfs/templates/security-configmap.yaml

@@ -13,10 +13,38 @@ data:
   security.toml: |-
     # this file is read by master, volume server, and filer
 
+    {{- if .Values.global.securityConfig.jwtSigning.volumeWrite }}
     # the jwt signing key is read by master and volume server
     # a jwt expires in 10 seconds
     [jwt.signing]
     key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
+
+    {{- if .Values.global.securityConfig.jwtSigning.volumeRead }}
+    # this jwt signing key is read by master and volume server, and it is used for read operations:
+    # - the Master server generates the JWT, which can be used to read a certain file on a volume server
+    # - the Volume server validates the JWT on reading
+    [jwt.signing.read]
+    key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
+
+    {{- if .Values.global.securityConfig.jwtSigning.filerWrite }}
+    # If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
+    # - f.e. the S3 API Shim generates the JWT
+    # - the Filer server validates the JWT on writing
+    # the jwt defaults to expire after 10 seconds.
+    [jwt.filer_signing]
+    key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
+
+    {{- if .Values.global.securityConfig.jwtSigning.filerRead }}
+    # If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
+    # - f.e. the S3 API Shim generates the JWT
+    # - the Filer server validates the JWT on writing
+    # the jwt defaults to expire after 10 seconds.
+    [jwt.filer_signing.read]
+    key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
 
     # all grpc tls authentications are mutual
     # the values for the following ca, cert, and key are paths to the PERM files.

k8s/charts/seaweedfs/templates/service-account.yaml

@@ -4,7 +4,7 @@
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: seaweedfs-rw-cr
+  name: {{ .Values.global.serviceAccountName }}-rw-cr
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
@@ -18,7 +18,7 @@ rules:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: seaweedfs-rw-sa
+  name: {{ .Values.global.serviceAccountName }}
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
@@ -29,7 +29,7 @@ metadata:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: system:serviceaccount:seaweedfs-rw-sa:default
+  name: system:serviceaccount:{{ .Values.global.serviceAccountName }}:default
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
@@ -37,10 +37,10 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
 subjects:
 - kind: ServiceAccount
-  name: seaweedfs-rw-sa
+  name: {{ .Values.global.serviceAccountName }}
   namespace: {{ .Release.Namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: seaweedfs-rw-cr
+  name: {{ .Values.global.serviceAccountName }}-rw-cr
 {{- end }}

k8s/charts/seaweedfs/templates/volume-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/volume-service.yaml

@@ -10,6 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
   clusterIP: None
+  internalTrafficPolicy: {{ .Values.volume.internalTrafficPolicy | default "Cluster" }}
   ports:
   - name: "swfs-volume"
     port: {{ .Values.volume.port }}
@@ -27,4 +28,4 @@ spec:
 {{- end }}
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-    app.kubernetes.io/component: volume
+    app.kubernetes.io/component: volume

k8s/charts/seaweedfs/templates/volume-statefulset.yaml

@@ -12,7 +12,7 @@ metadata:
 spec:
   serviceName: {{ template "seaweedfs.name" . }}-volume
   replicas: {{ .Values.volume.replicas }}
-  podManagementPolicy: Parallel
+  podManagementPolicy: {{ .Values.volume.podManagementPolicy }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
@@ -26,6 +26,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: volume
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       {{- if .Values.volume.affinity }}
       affinity:
@@ -42,8 +49,8 @@ spec:
       priorityClassName: {{ .Values.volume.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if .Values.volume.serviceAccountName }}
-      serviceAccountName: {{ .Values.volume.serviceAccountName | quote }}
+      {{- if .Values.global.createClusterRole }}
+      serviceAccountName: {{ .Values.volume.serviceAccountName | default .Values.global.serviceAccountName | quote }} # for deleting statefulset pods after migration
       {{- end }}
       {{- $initContainers_exists := include "volume.initContainers_exists" . -}}
       {{- if $initContainers_exists }}
@@ -178,7 +185,7 @@ spec:
             - containerPort: {{ .Values.volume.port }}
               name: swfs-vol
             {{- if .Values.volume.metricsPort }}
-            - containerPort: {{ .Values.filer.metricsPort }}
+            - containerPort: {{ .Values.volume.metricsPort }}
               name: metrics
             {{- end }}
             - containerPort: {{ .Values.volume.grpcPort }}
@@ -211,8 +218,9 @@ spec:
           resources:
             {{ tpl .Values.volume.resources . | nindent 12 | trim }}
           {{- end }}
-      {{- $hostpath_exists := include "volume.hostpath_exists" . -}}
-      {{- if $hostpath_exists }}
+      {{- if .Values.volume.sidecars }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }}
+      {{- end }}
       volumes:
         {{- if eq .Values.volume.data.type "hostPath" }}
         - name: data
@@ -220,18 +228,33 @@ spec:
             path: {{ .Values.volume.data.hostPathPrefix }}/object_store/
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.volume.data.type "existingClaim" }}
+        - name: data
+          persistentVolumeClaim:
+            claimName: {{ .Values.volume.data.claimName }} 
+        {{- end }}
         {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
         - name: idx
           hostPath:
             path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.volume.idx.type "existingClaim" }}
+        - name: idx
+          persistentVolumeClaim:
+            claimName: {{ .Values.volume.idx.claimName }} 
+        {{- end }}
         {{- if eq .Values.volume.logs.type "hostPath" }}
         - name: logs
           hostPath:
             path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.volume.logs.type "existingClaim" }}
+        - name: logs
+          persistentVolumeClaim:
+            claimName: {{ .Values.volume.data.claimName }} 
+        {{- end }}
         {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:
@@ -255,7 +278,6 @@ spec:
       {{- if .Values.volume.extraVolumes }}
         {{ tpl .Values.volume.extraVolumes . | indent 8 | trim }}
       {{- end }}
-      {{- end }}
       {{- if .Values.volume.nodeSelector }}
       nodeSelector:
         {{ tpl .Values.volume.nodeSelector . | indent 8 | trim }}
@@ -266,6 +288,10 @@ spec:
     {{- if eq .Values.volume.data.type "persistentVolumeClaim"}}
     - metadata:
         name: data
+        {{- with .Values.volume.data.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.volume.data.storageClass }}
@@ -276,6 +302,10 @@ spec:
     {{- if and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx }}
     - metadata:
         name: idx
+        {{- with .Values.volume.idx.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.volume.idx.storageClass }}
@@ -286,6 +316,10 @@ spec:
     {{- if eq .Values.volume.logs.type "persistentVolumeClaim" }}
     - metadata:
         name: logs
+        {{- with .Values.volume.logs.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.volume.logs.storageClass }}

k8s/charts/seaweedfs/values.yaml

@@ -10,6 +10,14 @@ global:
   restartPolicy: Always
   loggingLevel: 1
   enableSecurity: false
+  securityConfig:
+    jwtSigning:
+      volumeWrite: true
+      volumeRead: false
+      filerWrite: false
+      filerRead: false
+  # we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings
+  serviceAccountName: "seaweedfs"
   certificates:
     alphacrds: false
   monitoring:
@@ -66,11 +74,19 @@ master:
     # Enter any extra configuration for master.toml here.
     # It may be be a multi-line string.
 
-  # can use ANY storage-class , example with local-path-provisioner
+  # You may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
   #  data:
   #    type: "persistentVolumeClaim"
   #    size: "24Ti"
   #    storageClass: "local-path-provisioner"
+  #    annotations:
+  #      "key": "value"
+  #
+  # You may also spacify an existing claim:
+  #  data:
+  #    type: "existingClaim"
+  #    claimName: "my-pvc"
   data:
     type: "hostPath"
     size: ""
@@ -82,12 +98,26 @@ master:
     size: ""
     storageClass: ""
     hostPathPrefix: /storage
-
+  
+  ## @param master.sidecars Add additional sidecar containers to the master pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  ## Set podManagementPolicy
+  podManagementPolicy: Parallel
+
   # Resource requests, limits, etc. for the master cluster placement. This
   # should map directly to the value of the resources field for a PodSpec,
   # formatted as a multi-line string. By default no direct resource request
@@ -120,7 +150,7 @@ master:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-    beta.kubernetes.io/arch: amd64
+    kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -153,6 +183,7 @@ master:
         sub_filter '=/' '=./';
         sub_filter '/seaweedfsstatic' './seaweedfsstatic';
         sub_filter_once off;
+    tls: []
 
   extraEnvironmentVars:
     WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
@@ -208,11 +239,20 @@ volume:
   # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
   minFreeSpacePercent: 7
 
-# can use ANY storage-class , example with local-path-provisioner
-#  data:
-#    type: "persistentVolumeClaim"
-#    size: "24Ti"
-#    storageClass: "local-path-provisioner"
+  # You may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
+  #  data:
+  #    type: "persistentVolumeClaim"
+  #    size: "24Ti"
+  #    storageClass: "local-path-provisioner"
+  #    annotations:
+  #      "key": "value"
+  #
+  # You may also spacify an existing claim:
+  #  data:
+  #    type: "existingClaim"
+  #    claimName: "my-pvc"
+
   data:
     type: "hostPath"
     size: ""
@@ -257,12 +297,26 @@ volume:
 
   # Adjust jpg orientation when uploading.
   imagesFixOrientation: false
-
+  
+  ## @param volume.sidecars Add additional sidecar containers to the volume pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  ## Set podManagementPolicy
+  podManagementPolicy: Parallel
+
   # Affinity Settings
   # Commenting out or setting as empty the affinity variable, will allow
   # deployment to single node services such as Minikube
@@ -291,7 +345,7 @@ volume:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-   beta.kubernetes.io/arch: amd64
+   kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-volume: "true"
 
@@ -373,12 +427,19 @@ filer:
   storage: 25Gi
   # storageClass is the class of storage which defaults to null (the Kube cluster will pick the default).
   storageClass: null
-
-  # can use ANY storage-class , example with local-path-provisioner
+  # You may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
   #  data:
   #    type: "persistentVolumeClaim"
   #    size: "24Ti"
   #    storageClass: "local-path-provisioner"
+  #    annotations:
+  #      "key": "value"
+  #
+  # You may also specify an existing claim:
+  #  data:
+  #    type: "existingClaim"
+  #    claimName: "my-pvc"
   data:
     type: "hostPath"
     size: ""
@@ -390,12 +451,26 @@ filer:
     size: ""
     storageClass: ""
     hostPathPrefix: /storage
-
+  
+  ## @param filer.sidecars Add additional sidecar containers to the filer pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  ## Set podManagementPolicy
+  podManagementPolicy: Parallel
+
   # Affinity Settings
   # Commenting out or setting as empty the affinity variable, will allow
   # deployment to single node services such as Minikube
@@ -428,7 +503,7 @@ filer:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-    beta.kubernetes.io/arch: amd64
+    kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -530,9 +605,19 @@ filer:
     # Suffix of the host name, {bucket}.{domainName}
     domainName: ""
     # enable user & permission to s3 (need to inject to all services)
-    enableAuth: false
+    enableAuth: true
     skipAuthSecretCreation: false
+    # set to the name of an existing kubernetes Secret with the s3 json config file
+    # should have a secret key called seaweedfs_s3_config with an inline json configure
+    existingConfigSecret: ""
     auditLogConfig: {}
+    # You may specify buckets to be created during the install process.
+    # Buckets may be exposed publicly by setting `anonymousRead` to `true`
+    # createBuckets:
+    #   - name: bucket-a
+    #     anonymousRead: true
+    #   - name: bucket-b
+    #     anonymousRead: false
 
 s3:
   enabled: false
@@ -552,11 +637,25 @@ s3:
   # enable user & permission to s3 (need to inject to all services)
   enableAuth: false
   skipAuthSecretCreation: false
+  # set to the name of an existing kubernetes Secret with the s3 json config file
+  # should have a secret key called seaweedfs_s3_config with an inline json config
+  existingConfigSecret: false
   auditLogConfig: {}
 
   # Suffix of the host name, {bucket}.{domainName}
   domainName: ""
-
+  
+  ## @param s3.sidecars Add additional sidecar containers to the s3 pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
@@ -577,7 +676,7 @@ s3:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-    beta.kubernetes.io/arch: amd64
+    kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -621,6 +720,15 @@ s3:
     failureThreshold: 100
     timeoutSeconds: 10
 
+  ingress:
+    enabled: false
+    className: "nginx"
+    # host: false for "*" hostname
+    host: "seaweedfs.cluster.local"
+    # additional ingress annotations for the s3 endpoint
+    annotations: []
+    tls: []
+
 certificates:
   commonName: "SeaweedFS CA"
   ipAddresses: []
@@ -628,3 +736,13 @@ certificates:
   keySize: 2048
   duration: 2160h  # 90d
   renewBefore: 360h  # 15d
+  externalCertificates:
+    # This will avoid the need to use cert-manager and will rely on providing your own external certificates and CA
+    # you will need to store your provided certificates in the secret read by the different services:
+    # seaweedfs-master-cert, seaweedfs-filer-cert, etc. Can see any statefulset definition to see secret names
+    enabled: false
+
+# Labels to be added to all the created pods
+podLabels: {}
+# Annotations to be added to all the created pods
+podAnnotations: {}

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.55</version>
+    <version>3.59</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -16,8 +16,8 @@
     <properties>
         <protobuf.version>3.16.3</protobuf.version>
         <!-- follow https://github.com/grpc/grpc-java -->
-        <grpc.version>1.23.0</grpc.version>
-        <guava.version>30.0-jre</guava.version>
+        <grpc.version>1.53.0</grpc.version>
+        <guava.version>32.0.0-jre</guava.version>
     </properties>
 
     <dependencies>
@@ -96,6 +96,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.55</version>
+    <version>3.59</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -91,6 +91,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.55</version>
+    <version>3.59</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -90,6 +90,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -13,12 +13,12 @@ public class FilerClient extends FilerGrpcClient {
 
     private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
 
-    public FilerClient(String host, int grpcPort) {
-        super(host, grpcPort-10000, grpcPort);
+    public FilerClient(String filerHost, int filerGrpcPort) {
+        super(filerHost, filerGrpcPort-10000, filerGrpcPort);
     }
 
-    public FilerClient(String host, int port, int grpcPort) {
-        super(host, port, grpcPort);
+    public FilerClient(String filerHost, int filerPort, int filerGrpcPort) {
+        super(filerHost, filerPort, filerGrpcPort);
     }
 
     public static String toFileId(FilerProto.FileId fid) {

other/java/client/src/main/proto/filer.proto

@@ -424,6 +424,7 @@ message FilerConf {
         string data_center = 9;
         string rack = 10;
         string data_node = 11;
+        uint32 max_file_name_length = 12;
     }
     repeated PathConf locations = 2;
 }

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>3.55</version>
+            <version>3.59</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>3.55</version>
+            <version>3.59</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -17,6 +17,7 @@
         <configuration>
           <source>8</source>
           <target>8</target>
+          <release>8</release>
         </configuration>
       </plugin>
       <plugin>
@@ -317,7 +318,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>3.55</seaweedfs.client.version>
+    <seaweedfs.client.version>3.59</seaweedfs.client.version>
     <hadoop.version>3.2.4</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>3.55</seaweedfs.client.version>
+        <seaweedfs.client.version>3.59</seaweedfs.client.version>
         <hadoop.version>3.2.4</hadoop.version>
     </properties>
 
@@ -33,6 +33,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/hdfs3/dependency-reduced-pom.xml

@@ -17,6 +17,7 @@
         <configuration>
           <source>8</source>
           <target>8</target>
+          <release>8</release>
         </configuration>
       </plugin>
       <plugin>
@@ -317,7 +318,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>3.55</seaweedfs.client.version>
+    <seaweedfs.client.version>3.59</seaweedfs.client.version>
     <hadoop.version>3.2.4</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>3.55</seaweedfs.client.version>
+        <seaweedfs.client.version>3.59</seaweedfs.client.version>
         <hadoop.version>3.2.4</hadoop.version>
     </properties>
 
@@ -33,6 +33,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

test/random_access/pom.xml

@@ -50,6 +50,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
         </plugins>

test/s3/compatibility/.gitignore

@@ -1,2 +1,8 @@
+# Test run scratch data
 /s3-tests
 /tmp
+
+# Test run outputs
+weed.log
+compat.raw.txt
+compat.summary.txt

test/s3/compatibility/Dockerfile

@@ -1,11 +1,21 @@
 # the tests only support python 3.6, not newer
-FROM ubuntu:latest
+#FROM ubuntu:latest
+FROM python:3.6.15-slim-buster
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
+# Installed required system deps
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
+
+# Install python deps
+RUN pip install virtualenv
+
+# Clone Ceph S3 tests
 RUN git clone https://github.com/ceph/s3-tests.git
+
 WORKDIR s3-tests
 
-# we pin a certain commit
+# Pin to a certain commit on ceph/s3-tests
+# https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
 RUN git checkout 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
 
 RUN ./bootstrap

test/s3/compatibility/Makefile

@@ -0,0 +1,9 @@
+.PHONY: all build-image test
+
+all: build-image test
+
+build-image:
+	./prepare.sh
+
+test:
+	./run.sh

test/s3/compatibility/README.md

@@ -1,13 +1,245 @@
 # Running S3 Compatibility tests against SeaweedFS
 
-This is using [the tests from CephFS](https://github.com/ceph/s3-tests).
+This is using [the tests from CephFS][s3-tests].
+
+[s3-tests]: https://github.com/ceph/s3-tests
 
 ## Prerequisites
 
-- have Docker installed
+- have [Docker][docker] installed
 - this has been executed on Mac. On Linux, the hostname in `s3tests.conf`  needs to be adjusted.
 
+[docker]: https://docs.docker.com
+
 ## Running tests
 
-- `./prepare.sh` to build the docker image
-- `./run.sh` to execute all tests
+To build the docker image that is used for the tests:
+
+```console
+./prepare.sh
+```
+
+To execute all tests:
+
+```console
+./run.sh
+```
+To see debug output including all commands run by the script:
+
+```console
+DEBUG=y ./run.sh
+```
+
+> [!WARNING]
+>
+> If your output does *not* look like the content in [`results.summary.txt`](./results.summary.txt)
+> and it is full of HTTP level exceptions, there is likely an error contacting the `weed` server from
+> the container that is runnin the S3 compatibility tests.
+>
+> There are at least a couple ways to solve this:
+>
+> - Modify your `docker` setup to ensure `host.docker.internal` is connected to your host running `weed`
+> - Use `--net=host` and modify `host` in `s3tests.conf` to `localhost`
+>
+> The `--net=host` solution is potentially *unsafe*, as the container running [s3-tests][s3-tests] could
+> visit unexpected websites or use the host-passthrough internet access maliciously.
+>
+> If you are OK with the risk of allowing `--net=host`:
+>
+> - Set `host = localhost` in `s3tests.conf`
+> - Set `DOCKER_NET_HOST=y` when running `run.sh`
+
+## Most recent results
+
+See [`results.summary.txt`](./results.summary.txt) for the latest results of compatibility testing (with the caveat that `s3-tests` is pinned to [`ceph/s3-tests` @ 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e](https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e)).
+
+The file is reproduced below for ease of access:
+
+```
+/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
+  warnings.warn(warning, PythonDeprecationWarning)
+s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
+s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
+s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
+s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
+s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
+s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
+s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
+s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
+s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
+s3tests_boto3.functional.test_s3.test_object_write_file ... ok
+s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
+s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
+s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
+s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
+s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
+s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
+s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
+s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
+s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
+s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
+s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
+s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
+ERROR
+
+======================================================================
+ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
+    self.test(*self.arg)
+  File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
+    'DisplayName': acl_response['Owner']['DisplayName'],
+KeyError: 'Owner'
+-------------------- >> begin captured logging << --------------------
+```

test/s3/compatibility/results.summary.txt

@@ -0,0 +1,186 @@
+/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
+  warnings.warn(warning, PythonDeprecationWarning)
+s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
+s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
+s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
+s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
+s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
+s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
+s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
+s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
+s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
+s3tests_boto3.functional.test_s3.test_object_write_file ... ok
+s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
+s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
+s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
+s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
+s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
+s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
+s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
+s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
+s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
+s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
+s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
+s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
+ERROR
+
+======================================================================
+ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
+    self.test(*self.arg)
+  File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
+    'DisplayName': acl_response['Owner']['DisplayName'],
+KeyError: 'Owner'
+-------------------- >> begin captured logging << --------------------

test/s3/compatibility/run.sh

@@ -1,24 +1,88 @@
 #!/usr/bin/env bash
 
-set -ex
+CONTAINER_NAME=${CONTAINER_NAME:-s3test-instance}
+CONF_FILE=${CONF_FILE:-s3tests.conf}
+WEED_BIN=${WEED_BIN:-../../../weed/weed}
+TEST_RAW_OUTPUT_FILE=${TEST_RAW_OUTPUT_FILE:-compat.raw.txt}
+TEST_PROCESSED_OUTPUT_FILE=${TEST_PROCESSED_OUTPUT_FILE:-compat.summary.txt}
 
+# Set up debugging for this bash script if DEBUG is set
+if [ -n "${DEBUG}" ]; then
+    echo -e "DEBUG set [${DEBUG}], enabling debugging output...";
+    set -ex
+fi
+
+# Reset from possible previous test run
 killall -9 weed || echo "already stopped"
 rm -Rf tmp
 mkdir tmp
-docker stop s3test-instance || echo "already stopped"
+docker stop $CONTAINER_NAME || echo "already stopped"
 
+# Ensure ulimit is set to reasonable value
 ulimit -n 10000
-../../../weed/weed server -filer -s3 -volume.max 0 -master.volumeSizeLimitMB 5 -dir "$(pwd)/tmp" 1>&2>weed.log &
 
+# Start weed w/ filer + s3 in the background
+$WEED_BIN server \
+          -filer \
+          -s3 \
+          -volume.max 0 \
+          -master.volumeSizeLimitMB 5 \
+          -dir "$(pwd)/tmp" \
+          1>&2>weed.log &
+
+# Wait for master to start up
+echo -e "\n[info] waiting for master @ 9333...";
 until curl --output /dev/null --silent --head --fail http://127.0.0.1:9333; do
-    printf '.'
-    sleep 5
+    printf '.';
+    sleep 5;
 done
-sleep 3
+sleep 3;
 
-rm -Rf logs-full.txt logs-summary.txt
-# docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py:test_get_obj_tagging -v  -a 'resource=object,!bucket-policy,!versioning,!encryption'
-docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v "$(pwd)"/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py -v  -a 'resource=object,!bucket-policy,!versioning,!encryption' | sed -n -e '/botocore.hooks/!p;//q' | tee logs-summary.txt
+# Wait for s3 to start up
+echo -e "\n[info] waiting for S3 @ 8333...";
+until curl --output /dev/null --silent --fail http://127.0.0.1:8333; do
+    printf '.';
+    sleep 5;
+done
+sleep 3;
 
-docker stop s3test-instance || echo "already stopped"
-killall -9 weed
+# Determine whether docker net
+DOCKER_NET_HOST_ARGS=""
+if [ -n "${DOCKER_NET_HOST}" ]; then
+    DOCKER_NET_HOST_ARGS="--net=host"
+    echo -e "\n[info] setting docker to het nost"
+fi
+
+echo -e "\n[warn] You may have to run with UNFILTERED=y to disable output filtering, if you get the broken pipe error";
+echo -e "\n[info] running tests with unfiltered output...";
+docker run \
+       --name $CONTAINER_NAME \
+       --rm \
+       ${DOCKER_NET_HOST_ARGS} \
+       -e S3TEST_CONF=$CONF_FILE \
+       -v "$(pwd)"/$CONF_FILE:/s3-tests/s3tests.conf \
+       -it \
+       s3tests \
+       ./virtualenv/bin/nosetests \
+       s3tests_boto3/functional/test_s3.py \
+       -v \
+       -a 'resource=object,!bucket-policy,!versioning,!encryption' \
+    | tee ${TEST_RAW_OUTPUT_FILE}
+
+# If the summary logs are present, process them
+if [ -f "${TEST_RAW_OUTPUT_FILE}" ]; then
+    cat ${TEST_RAW_OUTPUT_FILE} | sed -n -e '/botocore.hooks/!p;//q' | tee ${TEST_PROCESSED_OUTPUT_FILE}
+    echo -e "\n[info] ✅ Successfully wrote processed output @ [${TEST_PROCESSED_OUTPUT_FILE}]";
+    if [ -z "${TEST_KEEP_RAW_OUTPUT}" ]; then
+        echo -e "\n[info] removing test raw output file @ [${TEST_RAW_OUTPUT_FILE}] (to disable this, set TEST_KEEP_RAW_OUTPUT=y)...";
+        rm -rf ${TEST_RAW_OUTPUT_FILE};
+    fi
+else
+    echo -e "\n[warn] failed to find raw output @ [${TEST_RAW_OUTPUT_FILE}]";
+fi
+
+echo -e "\n[info] stopping [${CONTAINER_NAME}] container...";
+docker stop $CONTAINER_NAME || echo "[info] already stopped";
+
+echo -e "\n[info] stopping seaweedfs processes (all, via kill -9)...";
+killall -9 weed;

test/s3/compatibility/s3tests.conf

@@ -4,6 +4,14 @@
 # host set for rgw in vstart.sh
 host = host.docker.internal
 
+## NOTE: if running docker --net=host (e.x. `DOCKER_NET_HOST=y ./run.sh`)
+## then use 'localhost', so the container can more easily access your *local* weed instance
+##
+## This is *unsafe* -- it is possible that this script will do malicious
+## things with host-level access. only run this if you trust the programmatic content of
+## ceph/s3-tests (https://github.com/ceph/s3-tests) which will be using this file
+#host = localhost
+
 # port set for rgw in vstart.sh
 port = 8333
 

unmaintained/see_dat/see_dat.go

@@ -33,8 +33,8 @@ func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
 
 func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
 	t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
-	glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v",
-		*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t)
+	glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v name %s",
+		*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t, n.Name)
 	return nil
 }
 

weed/cluster/lock_client.go

@@ -57,15 +57,15 @@ func (lc *LockClient) StartLock(key string, owner string) (lock *LiveLock) {
 		lc:             lc,
 	}
 	go func() {
-		util.RetryForever("create lock:"+key, func() error {
+		util.RetryUntil("create lock:"+key, func() error {
 			errorMessage, err := lock.doLock(lock_manager.MaxDuration)
 			if err != nil {
-				glog.Infof("create lock %s: %s", key, err)
+				glog.V(0).Infof("create lock %s: %s", key, err)
 				time.Sleep(time.Second)
 				return err
 			}
 			if errorMessage != "" {
-				glog.Infof("create lock %s: %s", key, errorMessage)
+				glog.V(4).Infof("create lock %s: %s", key, errorMessage)
 				time.Sleep(time.Second)
 				return fmt.Errorf("%v", errorMessage)
 			}
@@ -73,7 +73,6 @@ func (lc *LockClient) StartLock(key string, owner string) (lock *LiveLock) {
 			return nil
 		}, func(err error) (shouldContinue bool) {
 			if err != nil {
-				glog.Warningf("create lock %s: %s", key, err)
 				time.Sleep(time.Second)
 			}
 			return lock.renewToken == ""
@@ -98,7 +97,7 @@ func (lc *LockClient) doNewLock(key string, lockDuration time.Duration, owner st
 		lockDuration = lc.maxLockDuration
 		needRenewal = true
 	}
-	util.RetryForever("create lock:"+key, func() error {
+	util.RetryUntil("create lock:"+key, func() error {
 		errorMessage, err := lock.doLock(lockDuration)
 		if err != nil {
 			time.Sleep(time.Second)
@@ -148,7 +147,7 @@ func (lc *LockClient) keepLock(lock *LiveLock) {
 		select {
 		case <-ticker:
 			// renew the lock if lock.expireAtNs is still greater than now
-			util.RetryForever("keep lock:"+lock.key, func() error {
+			util.RetryUntil("keep lock:"+lock.key, func() error {
 				lockDuration := time.Duration(lock.expireAtNs-time.Now().UnixNano()) * time.Nanosecond
 				if lockDuration > lc.maxLockDuration {
 					lockDuration = lc.maxLockDuration

weed/command/filer.go

@@ -61,6 +61,7 @@ type FilerOptions struct {
 	showUIDirectoryDelete   *bool
 	downloadMaxMBps         *int
 	diskType                *string
+	allowedOrigins          *string
 }
 
 func init() {
@@ -91,6 +92,7 @@ func init() {
 	f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
 	f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
 	f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -169,7 +171,7 @@ func runFiler(cmd *Command, args []string) bool {
 
 	go stats_collect.StartMetricsServer(*f.bindIp, *f.metricsHttpPort)
 
-	filerAddress := util.JoinHostPort(*f.ip, *f.port)
+	filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
 	startDelay := time.Duration(2)
 	if *filerStartS3 {
 		filerS3Options.filer = &filerAddress
@@ -229,6 +231,9 @@ func (fo *FilerOptions) startFiler() {
 	if *fo.bindIp == "" {
 		*fo.bindIp = *fo.ip
 	}
+	if *fo.allowedOrigins == "" {
+		*fo.allowedOrigins = "*"
+	}
 
 	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
 
@@ -253,6 +258,7 @@ func (fo *FilerOptions) startFiler() {
 		ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
 		DownloadMaxBytesPs:    int64(*fo.downloadMaxMBps) * 1024 * 1024,
 		DiskType:              *fo.diskType,
+		AllowedOrigins:        strings.Split(*fo.allowedOrigins, ","),
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/filer_backup.go

@@ -8,6 +8,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"regexp"
 	"time"
 )
 
@@ -16,8 +17,10 @@ type FilerBackupOptions struct {
 	filer           *string
 	path            *string
 	excludePaths    *string
+	excludeFileName *string
 	debug           *bool
 	proxyByFiler    *bool
+	doDeleteFiles   *bool
 	timeAgo         *time.Duration
 	retentionDays   *int
 }
@@ -31,11 +34,12 @@ func init() {
 	filerBackupOptions.filer = cmdFilerBackup.Flag.String("filer", "localhost:8888", "filer of one SeaweedFS cluster")
 	filerBackupOptions.path = cmdFilerBackup.Flag.String("filerPath", "/", "directory to sync on filer")
 	filerBackupOptions.excludePaths = cmdFilerBackup.Flag.String("filerExcludePaths", "", "exclude directories to sync on filer")
+	filerBackupOptions.excludeFileName = cmdFilerBackup.Flag.String("filerExcludeFileName", "", "exclude file names that match the regexp to sync on filer")
 	filerBackupOptions.proxyByFiler = cmdFilerBackup.Flag.Bool("filerProxy", false, "read and write file chunks by filer instead of volume servers")
+	filerBackupOptions.doDeleteFiles = cmdFilerBackup.Flag.Bool("doDeleteFiles", false, "delete files on the destination")
 	filerBackupOptions.debug = cmdFilerBackup.Flag.Bool("debug", false, "debug mode to print out received files")
 	filerBackupOptions.timeAgo = cmdFilerBackup.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
 	filerBackupOptions.retentionDays = cmdFilerBackup.Flag.Int("retentionDays", 0, "incremental backup retention days")
-
 }
 
 var cmdFilerBackup = &Command{
@@ -90,6 +94,13 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 	sourceFiler := pb.ServerAddress(*backupOption.filer)
 	sourcePath := *backupOption.path
 	excludePaths := util.StringSplit(*backupOption.excludePaths, ",")
+	var reExcludeFileName *regexp.Regexp
+	if *backupOption.excludeFileName != "" {
+		var err error
+		if reExcludeFileName, err = regexp.Compile(*backupOption.excludeFileName); err != nil {
+			return fmt.Errorf("error compile regexp %v for exclude file name: %+v", *backupOption.excludeFileName, err)
+		}
+	}
 	timeAgo := *backupOption.timeAgo
 	targetPath := dataSink.GetSinkToDirectory()
 	debug := *backupOption.debug
@@ -119,7 +130,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 		*backupOption.proxyByFiler)
 	dataSink.SetSourceFiler(filerSource)
 
-	processEventFn := genProcessFunction(sourcePath, targetPath, excludePaths, dataSink, debug)
+	processEventFn := genProcessFunction(sourcePath, targetPath, excludePaths, reExcludeFileName, dataSink, *backupOption.doDeleteFiles, debug)
 
 	processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
 		glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3))

weed/command/filer_copy.go

@@ -31,21 +31,22 @@ var (
 )
 
 type CopyOptions struct {
-	include          *string
-	replication      *string
-	collection       *string
-	ttl              *string
-	diskType         *string
-	maxMB            *int
-	masterClient     *wdclient.MasterClient
-	concurrentFiles  *int
-	concurrentChunks *int
-	grpcDialOption   grpc.DialOption
-	masters          []string
-	cipher           bool
-	ttlSec           int32
-	checkSize        *bool
-	verbose          *bool
+	include            *string
+	replication        *string
+	collection         *string
+	ttl                *string
+	diskType           *string
+	maxMB              *int
+	masterClient       *wdclient.MasterClient
+	concurrentFiles    *int
+	concurrentChunks   *int
+	grpcDialOption     grpc.DialOption
+	masters            []string
+	cipher             bool
+	ttlSec             int32
+	checkSize          *bool
+	verbose            *bool
+	volumeServerAccess *string
 }
 
 func init() {
@@ -61,6 +62,7 @@ func init() {
 	copy.concurrentChunks = cmdFilerCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
 	copy.checkSize = cmdFilerCopy.Flag.Bool("check.size", false, "copy when the target file size is different from the source file")
 	copy.verbose = cmdFilerCopy.Flag.Bool("verbose", false, "print out details during copying")
+	copy.volumeServerAccess = cmdFilerCopy.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl]")
 }
 
 var cmdFilerCopy = &Command{
@@ -580,6 +582,9 @@ func (worker *FileCopyWorker) WithFilerClient(streamingMode bool, fn func(filer_
 }
 
 func (worker *FileCopyWorker) AdjustedUrl(location *filer_pb.Location) string {
+	if *worker.options.volumeServerAccess == "publicUrl" {
+		return location.PublicUrl
+	}
 	return location.Url
 }
 

weed/command/filer_remote_gateway.go

@@ -111,7 +111,7 @@ func runFilerRemoteGateway(cmd *Command, args []string) bool {
 
 	// synchronize /buckets folder
 	fmt.Printf("synchronize buckets in %s ...\n", remoteGatewayOptions.bucketsDir)
-	util.RetryForever("filer.remote.sync buckets", func() error {
+	util.RetryUntil("filer.remote.sync buckets", func() error {
 		return remoteGatewayOptions.followBucketUpdatesAndUploadToRemote(filerSource)
 	}, func(err error) bool {
 		if err != nil {

weed/command/filer_remote_gateway_buckets.go

@@ -30,10 +30,20 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
 		return err
 	}
 
-	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
-		lastTime := time.Unix(0, lastTsNs)
-		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
-		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, lastTsNs)
+	processor := NewMetadataProcessor(eachEntryFunc, 128)
+
+	var lastLogTsNs = time.Now().UnixNano()
+	processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
+		processor.AddSyncJob(resp)
+		return nil
+	}, 3*time.Second, func(counter int64, lastTsNs int64) error {
+		if processor.processedTsWatermark == 0 {
+			return nil
+		}
+		now := time.Now().UnixNano()
+		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, processor.processedTsWatermark), float64(counter)/(float64(now-lastLogTsNs)/1e9))
+		lastLogTsNs = now
+		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, processor.processedTsWatermark)
 	})
 
 	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo)
@@ -224,7 +234,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			if writeErr != nil {
 				return writeErr
 			}
-			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+			return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 		}
 		if filer_pb.IsDelete(resp) {
 			if resp.Directory == option.bucketsDir {
@@ -288,7 +298,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 						if writeErr != nil {
 							return writeErr
 						}
-						return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+						return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 					}
 				}
 			}
@@ -325,7 +335,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				if writeErr != nil {
 					return writeErr
 				}
-				return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+				return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 			}
 		}
 

weed/command/filer_remote_sync.go

@@ -90,7 +90,7 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
 
 	if dir != "" {
 		fmt.Printf("synchronize %s to remote storage...\n", dir)
-		util.RetryForever("filer.remote.sync "+dir, func() error {
+		util.RetryUntil("filer.remote.sync "+dir, func() error {
 			return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
 		}, func(err error) bool {
 			if err != nil {

weed/command/filer_remote_sync_dir.go

@@ -28,7 +28,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
 		return fmt.Errorf("read mount info: %v", detectErr)
 	}
 
-	eachEntryFunc, err := makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
+	eachEntryFunc, err := option.makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
 	if err != nil {
 		return err
 	}
@@ -80,7 +80,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
 	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
 }
 
-func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
+func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
 	client, err := remote_storage.GetRemoteStorage(remoteStorage)
 	if err != nil {
 		return nil, err
@@ -152,7 +152,7 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 			if writeErr != nil {
 				return writeErr
 			}
-			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+			return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 		}
 		if filer_pb.IsDelete(resp) {
 			glog.V(2).Infof("delete: %+v", resp)
@@ -191,7 +191,7 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 			if writeErr != nil {
 				return writeErr
 			}
-			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+			return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 		}
 
 		return nil

weed/command/filer_replication.go

@@ -99,7 +99,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 		if m.OldEntry != nil && m.NewEntry == nil {
 			glog.V(1).Infof("delete: %s", key)
 		} else if m.OldEntry == nil && m.NewEntry != nil {
-			glog.V(1).Infof("   add: %s", key)
+			glog.V(1).Infof("add: %s", key)
 		} else {
 			glog.V(1).Infof("modify: %s", key)
 		}

weed/command/filer_sync.go

@@ -17,6 +17,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 	"google.golang.org/grpc"
 	"os"
+	"regexp"
 	"strings"
 	"time"
 )
@@ -250,7 +251,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 	filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, grpcDialOption, sinkWriteChunkByFiler)
 	filerSink.SetSourceFiler(filerSource)
 
-	persistEventFn := genProcessFunction(sourcePath, targetPath, sourceExcludePaths, filerSink, debug)
+	persistEventFn := genProcessFunction(sourcePath, targetPath, sourceExcludePaths, nil, filerSink, true, debug)
 
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
@@ -368,7 +369,7 @@ func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signature
 
 }
 
-func genProcessFunction(sourcePath string, targetPath string, excludePaths []string, dataSink sink.ReplicationSink, debug bool) func(resp *filer_pb.SubscribeMetadataResponse) error {
+func genProcessFunction(sourcePath string, targetPath string, excludePaths []string, reExcludeFileName *regexp.Regexp, dataSink sink.ReplicationSink, doDeleteFiles bool, debug bool) func(resp *filer_pb.SubscribeMetadataResponse) error {
 	// process function
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
@@ -393,16 +394,22 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 				return nil
 			}
 		}
+		if reExcludeFileName != nil && reExcludeFileName.MatchString(message.NewEntry.Name) {
+			return nil
+		}
+		if dataSink.IsIncremental() {
+			doDeleteFiles = false
+		}
 		// handle deletions
 		if filer_pb.IsDelete(resp) {
+			if doDeleteFiles {
+				return nil
+			}
 			if !strings.HasPrefix(string(sourceOldKey), sourcePath) {
 				return nil
 			}
 			key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
-			if !dataSink.IsIncremental() {
-				return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
-			}
-			return nil
+			return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
 		}
 
 		// handle new entries
@@ -428,7 +435,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			// old key is in the watched directory
 			if strings.HasPrefix(string(sourceNewKey), sourcePath) {
 				// new key is also in the watched directory
-				if !dataSink.IsIncremental() {
+				if doDeleteFiles {
 					oldKey := util.Join(targetPath, string(sourceOldKey)[len(sourcePath):])
 					message.NewParentPath = util.Join(targetPath, message.NewParentPath[len(sourcePath):])
 					foundExisting, err := dataSink.UpdateEntry(string(oldKey), message.OldEntry, message.NewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
@@ -451,7 +458,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 
 			} else {
 				// new key is outside of the watched directory
-				if !dataSink.IsIncremental() {
+				if doDeleteFiles {
 					key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
 					return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
 				}

weed/command/master.go

@@ -288,8 +288,8 @@ func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers strin
 }
 
 func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
-	slices.SortFunc(peers, func(a, b pb.ServerAddress) bool {
-		return strings.Compare(string(a), string(b)) < 0
+	slices.SortFunc(peers, func(a, b pb.ServerAddress) int {
+		return strings.Compare(string(a), string(b))
 	})
 	if len(peers) <= 0 {
 		return true

weed/command/mq_broker.go

@@ -78,6 +78,9 @@ func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
 		Ip:                 *mqBrokerOpt.ip,
 		Port:               *mqBrokerOpt.port,
 	}, grpcDialOption)
+	if err != nil {
+		glog.Fatalf("failed to create new message broker for queue server: %v", err)
+	}
 
 	// start grpc listener
 	grpcL, _, err := util.NewIpAndLocalListeners("", *mqBrokerOpt.port, 0)

weed/command/s3.go

@@ -3,11 +3,14 @@ package command
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
@@ -40,8 +43,11 @@ type S3Options struct {
 	portGrpc                  *int
 	config                    *string
 	domainName                *string
+	allowedOrigins            *string
 	tlsPrivateKey             *string
 	tlsCertificate            *string
+	tlsCACertificate          *string
+	tlsVerifyClientCert       *bool
 	metricsHttpPort           *int
 	allowEmptyFolder          *bool
 	allowDeleteBucketNotEmpty *bool
@@ -60,11 +66,14 @@ func init() {
 	s3StandaloneOptions.portHttps = cmdS3.Flag.Int("port.https", 0, "s3 server https listen port")
 	s3StandaloneOptions.portGrpc = cmdS3.Flag.Int("port.grpc", 0, "s3 server grpc listen port")
 	s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
+	s3StandaloneOptions.allowedOrigins = cmdS3.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
 	s3StandaloneOptions.dataCenter = cmdS3.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
 	s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file")
 	s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file")
 	s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
 	s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
+	s3StandaloneOptions.tlsCACertificate = cmdS3.Flag.String("cacert.file", "", "path to the TLS CA certificate file")
+	s3StandaloneOptions.tlsVerifyClientCert = cmdS3.Flag.Bool("tlsVerifyClientCert", false, "whether to verify the client's certificate")
 	s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 	s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders")
 	s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
@@ -214,6 +223,7 @@ func (s3opt *S3Options) startS3Server() bool {
 		Port:                      *s3opt.port,
 		Config:                    *s3opt.config,
 		DomainName:                *s3opt.domainName,
+		AllowedOrigins:            strings.Split(*s3opt.allowedOrigins, ","),
 		BucketsPath:               filerBucketsPath,
 		GrpcDialOption:            grpcDialOption,
 		AllowEmptyFolder:          *s3opt.allowEmptyFolder,
@@ -289,7 +299,27 @@ func (s3opt *S3Options) startS3Server() bool {
 		if s3opt.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil {
 			glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
 		}
-		httpS.TLSConfig = &tls.Config{GetCertificate: s3opt.GetCertificateWithUpdate}
+
+		caCertPool := x509.NewCertPool()
+		if *s3Options.tlsCACertificate != "" {
+			// load CA certificate file and add it to list of client CAs
+			caCertFile, err := ioutil.ReadFile(*s3opt.tlsCACertificate)
+			if err != nil {
+				glog.Fatalf("error reading CA certificate: %v", err)
+			}
+			caCertPool.AppendCertsFromPEM(caCertFile)
+		}
+
+		clientAuth := tls.NoClientCert
+		if *s3Options.tlsVerifyClientCert {
+			clientAuth = tls.RequireAndVerifyClientCert
+		}
+
+		httpS.TLSConfig = &tls.Config{
+			GetCertificate: s3opt.GetCertificateWithUpdate,
+			ClientAuth:     clientAuth,
+			ClientCAs:      caCertPool,
+		}
 		if *s3opt.portHttps == 0 {
 			glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
 			if s3ApiLocalListener != nil {

weed/command/scaffold/filer.toml

@@ -12,6 +12,7 @@
 # with http DELETE, by default the filer would check whether a folder is empty.
 # recursive_delete will delete all sub folders and files, similar to "rm -Rf"
 recursive_delete = false
+#max_file_name_length = 255
 
 ####################################################
 # The following are filer store options
@@ -50,6 +51,9 @@ dbFile = "./filer.db"                # sqlite db file
 # ) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
 
 enabled = false
+# dsn will take priority over "hostname, port, username, password, database".
+# [username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
+dsn = "root@tcp(localhost:3306)/seaweedfs?collation=utf8mb4_bin"
 hostname = "localhost"
 port = 3306
 username = "root"

weed/command/scaffold/security.toml

@@ -4,6 +4,11 @@
 #    /etc/seaweedfs/security.toml
 # this file is read by master, volume server, and filer
 
+# comma separated origins allowed to make requests to the filer and s3 gateway. 
+# enter in this format: https://domain.com, or http://localhost:port
+[cors.allowed_origins]
+values = "*"
+
 # this jwt signing key is read by master and volume server, and it is used for write operations:
 # - the Master server generates the JWT, which can be used to write a certain file on a volume server
 # - the Volume server validates the JWT on writing

weed/command/server.go

@@ -106,6 +106,7 @@ func init() {
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
 	filerOptions.portGrpc = cmdServer.Flag.Int("filer.port.grpc", 0, "filer server grpc listen port")
 	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
+	filerOptions.allowedOrigins = cmdServer.Flag.String("filer.allowedOrigins", "*", "comma separated list of allowed origins")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
 	filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 4, "split files larger than the limit")
@@ -142,8 +143,11 @@ func init() {
 	s3Options.portHttps = cmdServer.Flag.Int("s3.port.https", 0, "s3 server https listen port")
 	s3Options.portGrpc = cmdServer.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
 	s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
+	s3Options.allowedOrigins = cmdServer.Flag.String("s3.allowedOrigins", "*", "comma separated list of allowed origins")
 	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
+	s3Options.tlsCACertificate = cmdServer.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
+	s3Options.tlsVerifyClientCert = cmdServer.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")
 	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
 	s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
 	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")

weed/command/volume.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/server/constants"
 	"github.com/seaweedfs/seaweedfs/weed/util/httpdown"
 
 	"google.golang.org/grpc/reflection"
@@ -241,7 +242,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
 		*v.idxFolder,
 		volumeNeedleMapKind,
-		v.masters, 5, *v.dataCenter, *v.rack,
+		v.masters, constants.VolumePulseSeconds, *v.dataCenter, *v.rack,
 		v.whiteList,
 		*v.fixJpgOrientation, *v.readMode,
 		*v.compactionMBPerSecond,

weed/filer/abstract_sql/abstract_sql_store.go

@@ -161,31 +161,24 @@ func (store *AbstractSqlStore) InsertEntry(ctx context.Context, entry *filer.Ent
 	if len(entry.GetChunks()) > filer.CountEntryChunksForGzip {
 		meta = util.MaybeGzipData(meta)
 	}
-
+	sqlInsert := "insert"
 	res, err := db.ExecContext(ctx, store.GetSqlInsert(bucket), util.HashStringToLong(dir), name, dir, meta)
-	if err == nil {
-		return
+	if err != nil && strings.Contains(strings.ToLower(err.Error()), "duplicate entry") {
+		// now the insert failed possibly due to duplication constraints
+		sqlInsert = "falls back to update"
+		glog.V(1).Infof("insert %s %s: %v", entry.FullPath, sqlInsert, err)
+		res, err = db.ExecContext(ctx, store.GetSqlUpdate(bucket), meta, util.HashStringToLong(dir), name, dir)
 	}
-
-	if !strings.Contains(strings.ToLower(err.Error()), "duplicate") {
-		// return fmt.Errorf("insert: %s", err)
-		// skip this since the error can be in a different language
-	}
-
-	// now the insert failed possibly due to duplication constraints
-	glog.V(1).Infof("insert %s falls back to update: %v", entry.FullPath, err)
-
-	res, err = db.ExecContext(ctx, store.GetSqlUpdate(bucket), meta, util.HashStringToLong(dir), name, dir)
 	if err != nil {
-		return fmt.Errorf("upsert %s: %s", entry.FullPath, err)
+		return fmt.Errorf("%s %s: %s", sqlInsert, entry.FullPath, err)
 	}
 
 	_, err = res.RowsAffected()
 	if err != nil {
-		return fmt.Errorf("upsert %s but no rows affected: %s", entry.FullPath, err)
+		return fmt.Errorf("%s %s but no rows affected: %s", sqlInsert, entry.FullPath, err)
 	}
-	return nil
 
+	return nil
 }
 
 func (store *AbstractSqlStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {

weed/filer/etcd/etcd_store.go

@@ -24,7 +24,7 @@ func init() {
 }
 
 type EtcdStore struct {
-	client *clientv3.Client
+	client        *clientv3.Client
 	etcdKeyPrefix string
 }
 
@@ -38,9 +38,9 @@ func (store *EtcdStore) Initialize(configuration weed_util.Configuration, prefix
 		servers = "localhost:2379"
 	}
 
-        username := configuration.GetString(prefix + "username")
-        password := configuration.GetString(prefix + "password")
-        store.etcdKeyPrefix = configuration.GetString(prefix + "key_prefix")
+	username := configuration.GetString(prefix + "username")
+	password := configuration.GetString(prefix + "password")
+	store.etcdKeyPrefix = configuration.GetString(prefix + "key_prefix")
 
 	timeout := configuration.GetString(prefix + "timeout")
 	if timeout == "" {
@@ -93,7 +93,7 @@ func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer.Entry) (er
 		meta = weed_util.MaybeGzipData(meta)
 	}
 
-	if _, err := store.client.Put(ctx, store.etcdKeyPrefix + string(key), string(meta)); err != nil {
+	if _, err := store.client.Put(ctx, store.etcdKeyPrefix+string(key), string(meta)); err != nil {
 		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
 	}
 
@@ -107,7 +107,7 @@ func (store *EtcdStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (er
 func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
 	key := genKey(fullpath.DirAndName())
 
-	resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(key))
+	resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(key))
 	if err != nil {
 		return nil, fmt.Errorf("get %s : %v", fullpath, err)
 	}
@@ -130,7 +130,7 @@ func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPa
 func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 	key := genKey(fullpath.DirAndName())
 
-	if _, err := store.client.Delete(ctx, store.etcdKeyPrefix + string(key)); err != nil {
+	if _, err := store.client.Delete(ctx, store.etcdKeyPrefix+string(key)); err != nil {
 		return fmt.Errorf("delete %s : %v", fullpath, err)
 	}
 
@@ -140,7 +140,7 @@ func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.Full
 func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
 	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
 
-	if _, err := store.client.Delete(ctx, store.etcdKeyPrefix + string(directoryPrefix), clientv3.WithPrefix()); err != nil {
+	if _, err := store.client.Delete(ctx, store.etcdKeyPrefix+string(directoryPrefix), clientv3.WithPrefix()); err != nil {
 		return fmt.Errorf("deleteFolderChildren %s : %v", fullpath, err)
 	}
 
@@ -158,7 +158,7 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u
 		lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
 	}
 
-	resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(lastFileStart),
+	resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(lastFileStart),
 		clientv3.WithFromKey(), clientv3.WithLimit(limit+1))
 	if err != nil {
 		return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)

weed/filer/etcd/etcd_store_kv.go

@@ -8,7 +8,7 @@ import (
 
 func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
 
-	_, err = store.client.Put(ctx, store.etcdKeyPrefix + string(key), string(value))
+	_, err = store.client.Put(ctx, store.etcdKeyPrefix+string(key), string(value))
 
 	if err != nil {
 		return fmt.Errorf("kv put: %v", err)
@@ -19,7 +19,7 @@ func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (er
 
 func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
 
-	resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(key))
+	resp, err := store.client.Get(ctx, store.etcdKeyPrefix+string(key))
 
 	if err != nil {
 		return nil, fmt.Errorf("kv get: %v", err)
@@ -34,7 +34,7 @@ func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, er
 
 func (store *EtcdStore) KvDelete(ctx context.Context, key []byte) (err error) {
 
-	_, err = store.client.Delete(ctx, store.etcdKeyPrefix + string(key))
+	_, err = store.client.Delete(ctx, store.etcdKeyPrefix+string(key))
 
 	if err != nil {
 		return fmt.Errorf("kv delete: %v", err)

weed/filer/filechunks2_test.go

@@ -73,11 +73,11 @@ func TestCompactFileChunksRealCase(t *testing.T) {
 }
 
 func printChunks(name string, chunks []*filer_pb.FileChunk) {
-	slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
+	slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) int {
 		if a.Offset == b.Offset {
-			return a.ModifiedTsNs < b.ModifiedTsNs
+			return int(a.ModifiedTsNs - b.ModifiedTsNs)
 		}
-		return a.Offset < b.Offset
+		return int(a.Offset - b.Offset)
 	})
 	for _, chunk := range chunks {
 		glog.V(0).Infof("%s chunk %s [%10d,%10d)", name, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))

weed/filer/filechunks_read.go

@@ -30,14 +30,20 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOff
 			isStart: false,
 		})
 	}
-	slices.SortFunc(points, func(a, b *Point) bool {
+	slices.SortFunc(points, func(a, b *Point) int {
 		if a.x != b.x {
-			return a.x < b.x
+			return int(a.x - b.x)
 		}
 		if a.ts != b.ts {
-			return a.ts < b.ts
+			return int(a.ts - b.ts)
 		}
-		return !a.isStart
+		if a.isStart {
+			return -1
+		}
+		if b.isStart {
+			return 1
+		}
+		return 0
 	})
 
 	var prevX int64

weed/filer/filer.go

@@ -3,12 +3,13 @@ package filer
 import (
 	"context"
 	"fmt"
-	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
 	"os"
 	"sort"
 	"strings"
 	"time"
 
+	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
+
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
@@ -50,9 +51,10 @@ type Filer struct {
 	FilerConf           *FilerConf
 	RemoteStorage       *FilerRemoteStorage
 	Dlm                 *lock_manager.DistributedLockManager
+	MaxFilenameLength   uint32
 }
 
-func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
+func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
 	f := &Filer{
 		MasterClient:        wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters),
 		fileIdDeletionQueue: util.NewUnboundedQueue(),
@@ -61,6 +63,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
 		RemoteStorage:       NewFilerRemoteStorage(),
 		UniqueFilerId:       util.RandomInt32(),
 		Dlm:                 lock_manager.NewDistributedLockManager(filerHost),
+		MaxFilenameLength:   maxFilenameLength,
 	}
 	if f.UniqueFilerId < 0 {
 		f.UniqueFilerId = -f.UniqueFilerId
@@ -194,12 +197,16 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {
 	return f.Store.RollbackTransaction(ctx)
 }
 
-func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool) error {
+func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool, maxFilenameLength uint32) error {
 
 	if string(entry.FullPath) == "/" {
 		return nil
 	}
 
+	if entry.FullPath.IsLongerFileName(maxFilenameLength) {
+		return fmt.Errorf("entry name too long")
+	}
+
 	oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
 
 	/*

weed/filer/filer_conf.go

@@ -166,6 +166,9 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
 		a.VolumeGrowthCount = b.VolumeGrowthCount
 	}
 	a.ReadOnly = b.ReadOnly || a.ReadOnly
+	if b.MaxFileNameLength > 0 {
+		a.MaxFileNameLength = b.MaxFileNameLength
+	}
 	a.DataCenter = util.Nvl(b.DataCenter, a.DataCenter)
 	a.Rack = util.Nvl(b.Rack, a.Rack)
 	a.DataNode = util.Nvl(b.DataNode, a.DataNode)