3.63

https://github.com/seaweedfs/seaweedfs/issues/5215

.github/workflows/binaries_dev.yml

@@ -38,13 +38,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -53,14 +53,14 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed-large-disk
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -68,7 +68,7 @@ jobs:
           release_tag: dev
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed-normal-disk
@@ -87,13 +87,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -102,14 +102,14 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed-large-disk
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -117,7 +117,7 @@ jobs:
           release_tag: dev
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed-normal-disk

.github/workflows/binaries_release0.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -38,13 +38,13 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -52,7 +52,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/binaries_release1.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -38,13 +38,13 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -52,7 +52,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/binaries_release2.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -38,13 +38,13 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -52,7 +52,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/binaries_release3.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -38,13 +38,13 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -52,7 +52,7 @@ jobs:
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/binaries_release4.yml

@@ -28,32 +28,32 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
           overwrite: true
-          build_flags: -tags elastic,ydb,gocdk,tikv
+          build_flags: -tags elastic,gocdk,sqlite,ydb,tikv,rclone
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@a4a9b2ed09837e8c8c59fcaeae64e5bba70d8d87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
-          build_flags: -tags 5BytesOffset,elastic,ydb,gocdk,tikv
-          ldflags: -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          build_flags: -tags 5BytesOffset,elastic,gocdk,sqlite,ydb,tikv,rclone
+          ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed
           binary_name: weed

.github/workflows/codeql.yml

@@ -18,11 +18,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         # Override language selection by uncommenting this and choosing your languages
         with:
           languages: go
@@ -30,7 +30,7 @@ jobs:
       # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below).
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -44,4 +44,4 @@ jobs:
       #     make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/container_dev.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -33,30 +33,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,30 +34,30 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,20 +34,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -35,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -35,20 +35,20 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,25 +34,25 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          build-args: TAGS=elastic,ydb,gocdk,tikv
+          build-args: TAGS=elastic,ydb,gocdk,tikv,rclone
           platforms: linux/amd64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release5.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -34,25 +34,25 @@ jobs:
             org.opencontainers.image.vendor=Chris Lu
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v1
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,tikv
+          build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,tikv,rclone
           platforms: linux/amd64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/depsreview.yml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@f6fff72a3217f580d5afd49a46826795305b63c7
+        uses: actions/dependency-review-action@fd07d42ce87ab09f10c61a2d1a5e59e6c655620a

.github/workflows/e2e.yml

@@ -21,16 +21,16 @@ jobs:
   e2e:
     name: FUSE Mount
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v2
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
 
     - name: Install dependencies
       run: |
@@ -41,20 +41,20 @@ jobs:
       timeout-minutes: 5
       run: make build_e2e && docker compose -f ./compose/e2e-mount.yml up --wait
 
-    - name: Run FIO
+    - name: Run FIO 4k
       timeout-minutes: 15
       run: |
         echo "Starting FIO at: $(date)"
         # Concurrent r/w
         echo 'Run randrw with size=16M bs=4k'
-        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1
+        docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randrw --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --group_reporting --runtime=30 --time_based=1
 
         echo "Verify FIO at: $(date)"
         # Verified write
         echo 'Run randwrite with size=16M bs=4k'
         docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=4k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
 
-    - name: Run FIO
+    - name: Run FIO 128k
       timeout-minutes: 15
       run: |
         echo "Starting FIO at: $(date)"
@@ -67,7 +67,7 @@ jobs:
         echo 'Run randwrite with size=16M bs=128k'
         docker compose -f ./compose/e2e-mount.yml exec mount timeout -k5 60 fio --name=fiotest --filename=/mnt/seaweedfs/fiotest --size=16M --rw=randwrite --bs=128k --direct=1 --numjobs=8 --ioengine=libaio --iodepth=32 --group_reporting --runtime=30 --time_based=1 --do_verify=0 --verify=crc32c --verify_backlog=1
 
-    - name: Run FIO
+    - name: Run FIO 1MB
       timeout-minutes: 15
       run: |
         echo "Starting FIO at: $(date)"
@@ -94,7 +94,7 @@ jobs:
 
     - name: Archive logs
       if: always()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: output-logs
         path: docker/output.log

.github/workflows/go.yml

@@ -21,20 +21,20 @@ jobs:
     steps:
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v2
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
 
     - name: Get dependencies
       run: |
         cd weed; go get -v -t -d ./...
 
     - name: Build
-      run: cd weed; go build -tags "elastic gocdk sqlite ydb tikv" -v .
+      run: cd weed; go build -tags "elastic gocdk sqlite ydb tikv rclone" -v .
 
     - name: Test
-      run: cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
+      run: cd weed; go test -tags "elastic gocdk sqlite ydb tikv rclone" -v ./...

.github/workflows/helm_chart_release.yml

@@ -1,7 +1,7 @@
 name: "helm: publish charts"
 on:
   push:
-    tags: 
+    tags:
       - '*'
 
 permissions:
@@ -12,7 +12,7 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
       - name: Publish Helm charts
         uses: stefanprodan/helm-gh-pages@master
         with:          

.github/workflows/helm_ci.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
         with:
           fetch-depth: 0
 
@@ -25,27 +25,27 @@ jobs:
         with:
           version: v3.10.0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           check-latest: true
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.4.0
+        uses: helm/chart-testing-action@v2.6.1
 
       - name: Run chart-testing (list-changed)
         id: list-changed
         run: |
-          changed=$(ct list-changed --chart-dirs k8s/charts)
+          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} --chart-dirs k8s/charts)
           if [[ -n "$changed" ]]; then
             echo "::set-output name=changed::true"
           fi
 
       - name: Run chart-testing (lint)
-        run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts
+        run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.8.0
+        uses: helm/kind-action@v1.9.0
 
       - name: Run chart-testing (install)
-        run: ct install --all --chart-dirs k8s/charts
+        run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts

.gitignore

@@ -87,3 +87,8 @@ other/java/hdfs/dependency-reduced-pom.xml
 
 # binary file
 weed/weed
+weed/mq/client/cmd/weed_pub/weed_pub
+docker/weed
+
+# test generated files
+weed/*/*.jpg

Makefile

@@ -1,14 +1,35 @@
 BINARY = weed
 
 SOURCE_DIR = .
+debug ?= 0
 
 all: install
 
 install:
 	cd weed; go install
 
-full_install:
-	cd weed; go install -tags "elastic gocdk sqlite ydb tikv"
+warp_install:
+	go install github.com/minio/warp@v0.7.6
 
-tests:
-	cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
+full_install:
+	cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone"
+
+server: install
+	weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json -metricsPort=9324
+
+benchmark: install warp_install
+	pkill weed || true
+	pkill warp || true
+	weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
+	warp client &
+	while ! nc -z localhost 8000 ; do sleep 1 ; done
+	warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm
+	pkill warp
+	pkill weed
+
+# curl -o profile "http://127.0.0.1:6060/debug/pprof/profile?debug=1"
+benchmark_with_pprof: debug = 1
+benchmark_with_pprof: benchmark
+
+test:
+	cd weed; go test -tags "elastic gocdk sqlite ydb tikv rclone" -v ./...

README.md

@@ -374,7 +374,7 @@ Each individual file size is limited to the volume size.
 
 ### Saving memory ###
 
-All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
+All file meta information stored on a volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
 
 ### Tiered Storage to the cloud ###
 
@@ -582,6 +582,71 @@ Percentage of the requests served within a certain time (ms)
   100%     54.1 ms
 ```
 
+### Run WARP and launch a mixed benchmark. ###
+
+```
+make benchmark
+warp: Benchmark data written to "warp-mixed-2023-10-16[102354]-l70a.csv.zst"                                                                                                                                                                                               
+Mixed operations.
+Operation: DELETE, 10%, Concurrency: 20, Ran 4m59s.
+ * Throughput: 6.19 obj/s
+
+Operation: GET, 45%, Concurrency: 20, Ran 5m0s.
+ * Throughput: 279.85 MiB/s, 27.99 obj/s
+
+Operation: PUT, 15%, Concurrency: 20, Ran 5m0s.
+ * Throughput: 89.86 MiB/s, 8.99 obj/s
+
+Operation: STAT, 30%, Concurrency: 20, Ran 5m0s.
+ * Throughput: 18.63 obj/s
+
+Cluster Total: 369.74 MiB/s, 61.79 obj/s, 0 errors over 5m0s.
+```
+
+To see segmented request statistics, use the --analyze.v parameter.
+```
+warp analyze --analyze.v warp-mixed-2023-10-16[102354]-l70a.csv.zst
+18642 operations loaded... Done!
+Mixed operations.
+----------------------------------------
+Operation: DELETE - total: 1854, 10.0%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
+ * Throughput: 6.19 obj/s
+
+Requests considered: 1855:
+ * Avg: 104ms, 50%: 30ms, 90%: 207ms, 99%: 1.355s, Fastest: 1ms, Slowest: 4.613s, StdDev: 320ms
+
+----------------------------------------
+Operation: GET - total: 8388, 45.3%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.12 +0500 +05
+ * Throughput: 279.77 MiB/s, 27.98 obj/s
+
+Requests considered: 8389:
+ * Avg: 221ms, 50%: 106ms, 90%: 492ms, 99%: 1.739s, Fastest: 8ms, Slowest: 8.633s, StdDev: 383ms
+ * TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 171ms, 99th: 669ms, Worst: 4.783s StdDev: 163ms
+ * First Access: Avg: 240ms, 50%: 105ms, 90%: 511ms, 99%: 2.08s, Fastest: 12ms, Slowest: 8.633s, StdDev: 480ms
+ * First Access TTFB: Avg: 88ms, Best: 2ms, 25th: 24ms, Median: 38ms, 75th: 64ms, 90th: 179ms, 99th: 919ms, Worst: 4.783s StdDev: 199ms
+ * Last Access: Avg: 219ms, 50%: 106ms, 90%: 463ms, 99%: 1.782s, Fastest: 9ms, Slowest: 8.633s, StdDev: 416ms
+ * Last Access TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 161ms, 99th: 657ms, Worst: 4.783s StdDev: 176ms
+
+----------------------------------------
+Operation: PUT - total: 2688, 14.5%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
+ * Throughput: 89.83 MiB/s, 8.98 obj/s
+
+Requests considered: 2689:
+ * Avg: 1.165s, 50%: 878ms, 90%: 2.015s, 99%: 5.74s, Fastest: 99ms, Slowest: 8.264s, StdDev: 968ms
+
+----------------------------------------
+Operation: STAT - total: 5586, 30.2%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.113 +0500 +05
+ * Throughput: 18.63 obj/s
+
+Requests considered: 5587:
+ * Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 80ms, Fastest: 0s, Slowest: 245ms, StdDev: 17ms
+ * First Access: Avg: 14ms, 50%: 10ms, 90%: 33ms, 99%: 69ms, Fastest: 0s, Slowest: 203ms, StdDev: 16ms
+ * Last Access: Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 74ms, Fastest: 0s, Slowest: 203ms, StdDev: 17ms
+
+Cluster Total: 369.64 MiB/s, 61.77 obj/s, 0 errors over 5m0s.
+Total Errors:0.
+```
+
 [Back to TOC](#table-of-contents)
 
 ## License ##

docker/Dockerfile.gccgo_build

@@ -1,44 +0,0 @@
-FROM gcc:11 as builder
-RUN mkdir -p /go/src/github.com/seaweedfs/
-RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
-ARG BRANCH=${BRANCH:-master}
-RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
-RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
-  && apt-get update \
-  && apt-get install -y golang-src \
-  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
-  && CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm
-
-FROM alpine AS final
-LABEL author="Chris Lu"
-COPY --from=builder /go/bin/weed /usr/bin/
-RUN mkdir -p /etc/seaweedfs
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
-RUN apk add fuse # for weed mount
-
-# volume server gprc port
-EXPOSE 18080
-# volume server http port
-EXPOSE 8080
-# filer server gprc port
-EXPOSE 18888
-# filer server http port
-EXPOSE 8888
-# master server shared gprc port
-EXPOSE 19333
-# master server shared http port
-EXPOSE 9333
-# s3 server http port
-EXPOSE 8333
-# webdav server http port
-EXPOSE 7333
-
-RUN mkdir -p /data/filerldb2
-
-VOLUME /data
-WORKDIR /data
-
-RUN chmod +x /entrypoint.sh
-
-ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.go_build

@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine as builder
+FROM golang:1.22-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/seaweedfs/
 RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs

docker/Dockerfile.rocksdb_dev_env

@@ -1,9 +1,9 @@
-FROM golang:1.20-buster as builder
+FROM golang:1.22 as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v8.1.1
+ENV ROCKSDB_VERSION v8.10.0
 
 # build RocksDB
 RUN cd /tmp && \

docker/Dockerfile.rocksdb_large

@@ -1,9 +1,9 @@
-FROM golang:1.20-buster as builder
+FROM golang:1.22 as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v8.1.1
+ENV ROCKSDB_VERSION v8.10.0
 
 # build RocksDB
 RUN cd /tmp && \

docker/Makefile

@@ -8,7 +8,7 @@ cgo ?= 0
 binary:
 	export SWCOMMIT=$(shell git rev-parse --short HEAD)
 	export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
-	cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
+	cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w -extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
 
 binary_race: options = -race
 binary_race: cgo = 1
@@ -102,12 +102,12 @@ clean:
 	rm ./weed
 
 certstrap:
-	go get github.com/square/certstrap
-	certstrap --depot-path compose/tls init --passphrase "" --common-name "SeaweedFS CA" || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name volume01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name master01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name filer01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name client01.dev || true
+	go install -v github.com/square/certstrap@latest
+	certstrap --depot-path compose/tls init --curve P-256 --passphrase "" --common-name "SeaweedFS CA" || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --domain localhost --common-name volume01.dev || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name master01.dev || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name filer01.dev || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name client01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true

docker/compose/local-mq-test.yml

@@ -0,0 +1,27 @@
+version: '3.9'
+
+services:
+  server:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+      - 8888:8888
+      - 18888:18888
+    command: "server -ip=server -filer -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1"
+    healthcheck:
+      test: curl -f http://localhost:8888/healthz
+  mq_broker:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 17777:17777
+    command: "mq.broker -master=server:9333 -ip=mq_broker"
+    depends_on:
+      server:
+        condition: service_healthy
+  mq_client:
+    image: chrislusf/seaweedfs:local
+    # run a custom command instead of entrypoint
+    command: "ls -al"
+    depends_on:
+      - mq_broker

docker/compose/local-sync-mount-compose.yml

@@ -3,19 +3,54 @@ services:
   node1:
     image: chrislusf/seaweedfs:local
     command: "server -master -volume -filer"
+    ports:
+      - 8888:8888
+      - 18888:18888
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
+      interval: 1s
+      start_period: 10s
+      timeout: 30s
   mount1:
     image: chrislusf/seaweedfs:local
     privileged: true
     command: "mount -filer=node1:8888 -dir=/mnt -dirAutoCreate"
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://node1:8888/" ]
+      interval: 1s
+      start_period: 10s
+      timeout: 30s
+    depends_on:
+      node1:
+        condition: service_healthy
   node2:
     image: chrislusf/seaweedfs:local
     ports:
       - 7888:8888
+      - 17888:18888
     command: "server -master -volume -filer"
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
+      interval: 1s
+      start_period: 10s
+      timeout: 30s
   mount2:
     image: chrislusf/seaweedfs:local
     privileged: true
     command: "mount -filer=node2:8888 -dir=/mnt -dirAutoCreate"
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://node2:8888/" ]
+      interval: 1s
+      start_period: 10s
+      timeout: 30s
+    depends_on:
+      node2:
+        condition: service_healthy
   sync:
     image: chrislusf/seaweedfs:local
     command: "-v=4 filer.sync -a=node1:8888 -b=node2:8888 -a.debug -b.debug"
+    depends_on:
+      mount1:
+        condition: service_healthy
+      mount2:
+        condition: service_healthy

docker/compose/s3.json

@@ -40,7 +40,10 @@
         "List",
         "Tagging",
         "Write"
-      ]
+      ],
+      "account": {
+        "id": "testid"
+      }
     },
     {
       "name": "s3_tests_alt",
@@ -101,5 +104,12 @@
         "Write"
       ]
     }
-  ]
+  ],
+  "accounts": [
+      {
+        "id" : "testid",
+        "displayName": "M. Tester",
+        "emailAddress": "tester@ceph.com"
+      }
+    ]
 }

docker/compose/tls.env

@@ -12,5 +12,9 @@ WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,clie
 WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
 WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
 WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
+WEED_HTTPS_CLIENT_ENABLE=true
+WEED_HTTPS_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt
+WEED_HTTPS_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key
+WEED_HTTPS_VOLUME_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt
 #GRPC_GO_LOG_SEVERITY_LEVEL=info
 #GRPC_GO_LOG_VERBOSITY_LEVEL=2

go.mod

@@ -1,22 +1,22 @@
 module github.com/seaweedfs/seaweedfs
 
-go 1.20
+go 1.22.0
 
 require (
-	cloud.google.com/go v0.110.7 // indirect
-	cloud.google.com/go/pubsub v1.33.0
-	cloud.google.com/go/storage v1.32.0
+	cloud.google.com/go v0.112.0 // indirect
+	cloud.google.com/go/pubsub v1.36.1
+	cloud.google.com/go/storage v1.36.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.38.1
-	github.com/aws/aws-sdk-go v1.44.314
+	github.com/aws/aws-sdk-go v1.50.10
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.1
@@ -29,10 +29,10 @@ require (
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fclairamb/ftpserverlib v0.22.0
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/go-redsync/redsync/v4 v4.8.2
+	github.com/go-redsync/redsync/v4 v4.11.0
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.6.0
@@ -41,11 +41,10 @@ require (
 	github.com/golang/protobuf v1.5.3
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/uuid v1.3.0
+	github.com/google/uuid v1.6.0
 	github.com/google/wire v0.5.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
-	github.com/gorilla/mux v1.8.0
+	github.com/gorilla/mux v1.8.1
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -58,42 +57,40 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
-	github.com/klauspost/reedsolomon v1.11.8
+	github.com/klauspost/compress v1.17.3 // indirect
+	github.com/klauspost/reedsolomon v1.12.1
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.8.0
+	github.com/linxGnu/grocksdb v1.8.12
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/peterh/liner v1.2.2
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.2.0
-	github.com/prometheus/client_golang v1.16.0
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.11.1
+	github.com/prometheus/client_golang v1.18.0
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
 	github.com/seaweedfs/raft v1.1.3
-	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/spf13/afero v1.9.5 // indirect
-	github.com/spf13/cast v1.5.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.16.0
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/viper v1.18.2
 	github.com/stretchr/testify v1.8.4
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tidwall/gjson v1.16.0
+	github.com/tidwall/gjson v1.17.0
 	github.com/tidwall/match v1.1.1
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
@@ -106,213 +103,250 @@ require (
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	go.etcd.io/etcd/client/v3 v3.5.9
-	go.mongodb.org/mongo-driver v1.12.1
+	go.etcd.io/etcd/client/v3 v3.5.11
+	go.mongodb.org/mongo-driver v1.13.1
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.34.0
-	gocloud.dev/pubsub/natspubsub v0.33.0
-	gocloud.dev/pubsub/rabbitpubsub v0.33.0
-	golang.org/x/crypto v0.12.0 // indirect
-	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
-	golang.org/x/image v0.11.0
-	golang.org/x/net v0.14.0
-	golang.org/x/oauth2 v0.10.0 // indirect
-	golang.org/x/sys v0.12.0
-	golang.org/x/text v0.12.0 // indirect
-	golang.org/x/tools v0.11.0
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.134.0
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230731193218-e0aa005b6bdf // indirect
-	google.golang.org/grpc v1.57.0
-	google.golang.org/protobuf v1.31.0
+	gocloud.dev v0.36.0
+	gocloud.dev/pubsub/natspubsub v0.36.0
+	gocloud.dev/pubsub/rabbitpubsub v0.36.0
+	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/image v0.15.0
+	golang.org/x/net v0.21.0
+	golang.org/x/oauth2 v0.17.0 // indirect
+	golang.org/x/sys v0.17.0
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.16.1
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/api v0.165.0
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect
+	google.golang.org/grpc v1.61.0
+	google.golang.org/protobuf v1.32.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect
 	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.24.1 // indirect
-	modernc.org/mathutil v1.5.0
-	modernc.org/memory v1.6.0 // indirect
+	modernc.org/libc v1.29.0 // indirect
+	modernc.org/mathutil v1.6.0
+	modernc.org/memory v1.7.2 // indirect
 	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sqlite v1.25.0
-	modernc.org/strutil v1.1.3
+	modernc.org/sqlite v1.28.0
+	modernc.org/strutil v1.2.0
 	modernc.org/token v1.0.1 // indirect
 )
 
 require (
-	github.com/Jille/raft-grpc-transport v1.4.0
-	github.com/arangodb/go-driver v1.6.0
+	github.com/Jille/raft-grpc-transport v1.5.0
+	github.com/arangodb/go-driver v1.6.1
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.21.0
-	github.com/aws/aws-sdk-go-v2/config v1.18.33
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.35
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.38.2
+	github.com/aws/aws-sdk-go-v2 v1.24.1
+	github.com/aws/aws-sdk-go-v2/config v1.26.6
+	github.com/aws/aws-sdk-go-v2/credentials v1.16.16
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0
 	github.com/fluent/fluent-logger-golang v1.9.0
-	github.com/golang-jwt/jwt/v5 v5.0.0
+	github.com/golang-jwt/jwt/v5 v5.1.0
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
-	github.com/hanwen/go-fuse/v2 v2.3.0
-	github.com/hashicorp/raft v1.5.0
-	github.com/hashicorp/raft-boltdb/v2 v2.2.2
+	github.com/hanwen/go-fuse/v2 v2.5.0
+	github.com/hashicorp/raft v1.6.0
+	github.com/hashicorp/raft-boltdb/v2 v2.3.0
 	github.com/orcaman/concurrent-map/v2 v2.0.1
-	github.com/puzpuzpuz/xsync/v2 v2.5.0
-	github.com/rabbitmq/amqp091-go v1.8.1
-	github.com/rclone/rclone v1.63.1
-	github.com/schollz/progressbar/v3 v3.13.1
+	github.com/puzpuzpuz/xsync/v2 v2.5.1
+	github.com/rabbitmq/amqp091-go v1.9.0
+	github.com/rclone/rclone v1.65.2
+	github.com/rdleal/intervalst v1.2.2
+	github.com/schollz/progressbar/v3 v3.14.1
+	github.com/shirou/gopsutil/v3 v3.24.1
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.48.8
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.56.1
+	go.uber.org/atomic v1.11.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
 
 require (
-	cloud.google.com/go/compute v1.23.0 // indirect
+	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 // indirect
+	cloud.google.com/go/iam v1.1.5 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
-	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
+	github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect
+	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
+	github.com/ProtonMail/go-srp v0.0.7 // indirect
+	github.com/ProtonMail/gopenpgp/v2 v2.7.4 // indirect
+	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/Unknwon/goconfig v1.0.0 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.12 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.33 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.13.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect
-	github.com/aws/smithy-go v1.14.2 // indirect
-	github.com/benbjohnson/clock v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.7.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.26.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.29.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
+	github.com/aws/smithy-go v1.19.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/bradenaw/juniper v0.13.1 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
-	github.com/colinmarc/hdfs/v2 v2.3.0 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 // indirect
+	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
+	github.com/cronokirby/saferith v0.33.0 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
+	github.com/emersion/go-message v0.17.0 // indirect
+	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
+	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
 	github.com/fatih/color v1.14.1 // indirect
 	github.com/fclairamb/go-log v0.4.1 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/flynn/noise v1.0.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
-	github.com/go-chi/chi/v5 v5.0.8 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-chi/chi/v5 v5.0.10 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-resty/resty/v2 v2.11.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/s2a-go v0.1.4 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-msgpack v1.1.5 // indirect
+	github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
-	github.com/hirochachacha/go-smb2 v1.1.0 // indirect
+	github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
+	github.com/henrybear327/go-proton-api v1.0.0 // indirect
 	github.com/iguanesolutions/go-systemd/v5 v5.1.1 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/jlaffaye/ftp v0.2.0 // indirect
 	github.com/jonboulle/clockwork v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
+	github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect
+	github.com/jtolio/noiseconn v0.0.0-20230621152802-afeab29449e0 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.7.0 // indirect
-	github.com/nats-io/nats.go v1.28.0 // indirect
-	github.com/nats-io/nkeys v0.4.4 // indirect
+	github.com/nats-io/nats.go v1.31.0 // indirect
+	github.com/nats-io/nkeys v0.4.6 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
-	github.com/ncw/swift/v2 v2.0.1 // indirect
+	github.com/ncw/swift/v2 v2.0.2 // indirect
+	github.com/onsi/gomega v1.27.10 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.34.0 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.51.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
-	github.com/philhofer/fwd v1.1.1 // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
 	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
-	github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 // indirect
+	github.com/pkg/sftp v1.13.6 // indirect
 	github.com/pkg/xattr v0.4.9 // indirect
-	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
-	github.com/rclone/ftp v0.0.0-20230327202000-dadc1f64e87d // indirect
+	github.com/relvacode/iso8601 v1.3.0 // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/shirou/gopsutil/v3 v3.23.5 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
 	github.com/sony/gobreaker v0.5.0 // indirect
-	github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca // indirect
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
-	github.com/tinylib/msgp v1.1.6 // indirect
-	github.com/tklauser/go-sysconf v0.3.11 // indirect
-	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222 // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf // indirect
 	github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
-	go.etcd.io/bbolt v1.3.7 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
+	go.etcd.io/bbolt v1.3.8 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.11 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.11 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
+	go.opentelemetry.io/otel v1.23.0 // indirect
+	go.opentelemetry.io/otel/metric v1.23.0 // indirect
+	go.opentelemetry.io/otel/trace v1.23.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.11.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230731193218-e0aa005b6bdf // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230731193218-e0aa005b6bdf // indirect
+	go.uber.org/zap v1.26.0 // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect
-	storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
-	storj.io/drpc v0.0.32 // indirect
-	storj.io/uplink v1.10.0 // indirect
+	storj.io/common v0.0.0-20231027080355-b4cb1b0d728e // indirect
+	storj.io/drpc v0.0.33 // indirect
+	storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
+	storj.io/uplink v1.12.1 // indirect
 )
 
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft

k8s/charts/seaweedfs/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.56"
-version: "3.56"
+appVersion: "3.63"
+version: 3.63.0

k8s/charts/seaweedfs/README.md

@@ -4,15 +4,21 @@
 
 ### Add the helm repo
 
-`helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm`
+```bash
+helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm
+```
 
 ### Install the helm chart
 
-`helm install seaweedfs seaweedfs/seaweedfs`
+```bash
+helm install seaweedfs seaweedfs/seaweedfs
+```
 
 ### (Recommended) Provide `values.yaml`
 
-`helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs`
+```bash
+helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs
+```
 
 ## Info:
 * master/filer/volume are stateful sets with anti-affinity on the hostname,
@@ -78,4 +84,63 @@ for your deployment.
 You can update the replicas count for each node type in values.yaml,
 need to add more nodes with the corresponding labels if applicable.
 
-Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
+Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
+
+## S3 configuration
+
+To enable an s3 endpoint for your filer with a default install add the following to your values.yaml:
+
+```yaml
+filer:
+  s3:
+    enabled: true
+```
+
+### Enabling Authenticaion to S3
+
+To enable authentication for S3, you have two options:
+
+- let the helm chart create an admin user as well as a read only user
+- provide your own s3 config.json file via an existing Kubernetes Secret
+
+#### Use the default credentials for S3
+
+Example parameters for your values.yaml:
+
+```yaml
+filer:
+  s3:
+    enabled: true
+    enableAuth: true
+```
+
+#### Provide your own credentials for S3
+
+Example parameters for your values.yaml:
+
+```yaml
+filer:
+  s3:
+    enabled: true
+    enableAuth: true
+    existingConfigSecret: my-s3-secret
+```
+
+Example existing secret with your s3 config to create an admin user and readonly user, both with credentials:
+
+```yaml
+---
+# Source: seaweedfs/templates/seaweedfs-s3-secret.yaml
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: my-s3-secret
+  namespace: seaweedfs
+  labels:
+    app.kubernetes.io/name: seaweedfs
+    app.kubernetes.io/component: s3
+stringData:
+  # this key must be an inline json config file
+  seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"snu8yoP6QAlY0ne4","secretKey":"PNzBcmeLNEdR0oviwm04NQAicOrDH1Km"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"SCigFee6c5lbi04A","secretKey":"kgFhbT38R8WUYVtiFQ1OiSVOrYr3NKku"}],"actions":["Read"]}]}'
+```

k8s/charts/seaweedfs/templates/_helpers.tpl

@@ -118,20 +118,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-
-{{/* check if any Volume HostPath exists */}}
-{{- define "volume.hostpath_exists" -}}
-{{- if or (or (eq .Values.volume.data.type "hostPath") (and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "hostPath") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- if or .Values.global.enableSecurity .Values.volume.extraVolumes -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
-{{- end -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
@@ -140,16 +127,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- if or (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-
-{{/* check if any Filer HostPath exists */}}
-{{- define "filer.hostpath_exists" -}}
-{{- if or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.logs.type "hostPath") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
@@ -158,29 +136,16 @@ Inject extra environment vars in the format key:value, if populated
 {{- if or (eq .Values.master.data.type "persistentVolumeClaim") (eq .Values.master.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-
-{{/* check if any Master HostPath exists */}}
-{{- define "master.hostpath_exists" -}}
-{{- if or (eq .Values.master.data.type "hostPath") (eq .Values.master.logs.type "hostPath") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- if or .Values.global.enableSecurity .Values.volume.extraVolumes -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
-{{- end -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
 {{/* check if any InitContainers exist for Volumes */}}
 {{- define "volume.initContainers_exists" -}}
-{{- if or (not (empty .Values.volume.dir_idx )) (not (empty .Values.volume.initContainers )) -}}
+{{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
@@ -198,3 +163,22 @@ imagePullSecrets:
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{/*
+Renders a value that contains template perhaps with scope if the scope is present.
+Usage:
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
+*/}}
+{{- define "common.tplvalues.render" -}}
+{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
+{{- if contains "{{" (toJson .value) }}
+  {{- if .scope }}
+      {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }}
+  {{- else }}
+    {{- tpl $value .context }}
+  {{- end }}
+{{- else }}
+    {{- $value }}
+{{- end }}
+{{- end -}}

k8s/charts/seaweedfs/templates/ca-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/cert-caissuer.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Issuer
 metadata:

k8s/charts/seaweedfs/templates/cert-clusterissuer.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: ClusterIssuer
 metadata:

k8s/charts/seaweedfs/templates/client-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/filer-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/filer-service-client.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.filer.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -32,3 +33,4 @@ spec:
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     app.kubernetes.io/component: filer
+{{- end }}

k8s/charts/seaweedfs/templates/filer-service.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.filer.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -44,3 +45,4 @@ spec:
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     app.kubernetes.io/component: filer
+{{- end }}

k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.filer.enabled }}
 {{- if .Values.filer.metricsPort }}
 {{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
@@ -21,4 +22,5 @@ spec:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
       app.kubernetes.io/component: filer
 {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -12,7 +12,7 @@ metadata:
     app.kubernetes.io/component: filer
 spec:
   serviceName: {{ template "seaweedfs.name" . }}-filer
-  podManagementPolicy: Parallel
+  podManagementPolicy: {{ .Values.filer.podManagementPolicy }}
   replicas: {{ .Values.filer.replicas }}
   {{- if (gt (int .Values.filer.updatePartition) 0) }}
   updateStrategy:
@@ -23,7 +23,6 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-      helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
       app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: filer
   template:
@@ -33,6 +32,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: filer
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.filer.restartPolicy }}
       {{- if .Values.filer.affinity }}
@@ -45,16 +51,13 @@ spec:
       {{- end }}
       {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
       {{- if .Values.global.createClusterRole }}
-      serviceAccountName: seaweedfs-rw-sa #hack for delete pod master after migration
+      serviceAccountName: {{ .Values.filer.serviceAccountName | default .Values.global.serviceAccountName | quote }} # for deleting statefulset pods after migration
       {{- end }}
       terminationGracePeriodSeconds: 60
       {{- if .Values.filer.priorityClassName }}
       priorityClassName: {{ .Values.filer.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if .Values.filer.serviceAccountName }}
-      serviceAccountName: {{ .Values.filer.serviceAccountName | quote }}
-      {{- end }}
       {{- if .Values.filer.initContainers }}
       initContainers:
         {{ tpl .Values.filer.initContainers . | nindent 8 | trim }}
@@ -102,7 +105,12 @@ spec:
             {{- if .Values.global.extraEnvironmentVars }}
             {{- range $key, $value := .Values.global.extraEnvironmentVars }}
             - name: {{ $key }}
+            {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
             {{- end }}
             {{- end }}
             {{- if .Values.filer.secretExtraEnvironmentVars }}
@@ -110,11 +118,11 @@ spec:
             - name: {{ $key }}
               valueFrom: {{ toYaml $value | nindent 16 }}
             {{- end }}
-            {{- end }}             
+            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
-            - | 
+            - |
               exec /usr/bin/weed \
               {{- if eq .Values.filer.logs.type "hostPath" }}
               -logdir=/logs \
@@ -182,15 +190,17 @@ spec:
               -s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
               {{- end }}
               {{- end }}
-              -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+              -master={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
           volumeMounts:
             {{- if eq .Values.filer.logs.type "hostPath" }}
             - name: seaweedfs-filer-log-volume
               mountPath: "/logs/"
             {{- end }}
-            - mountPath: /etc/sw
-              name: config-users
+            {{- if .Values.filer.s3.enableAuth }}
+            - name: config-users
+              mountPath: /etc/sw
               readOnly: true
+            {{- end }}
             {{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim"))) }}
             - name: data-filer
               mountPath: /data
@@ -252,6 +262,9 @@ spec:
           resources:
             {{ tpl .Values.filer.resources . | nindent 12 | trim }}
           {{- end }}
+        {{- if .Values.filer.sidecars }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
+        {{- end }}
       volumes:
         {{- if eq .Values.filer.logs.type "hostPath" }}
         - name: seaweedfs-filer-log-volume
@@ -259,19 +272,35 @@ spec:
             path: {{ .Values.filer.logs.hostPathPrefix }}/logs/seaweedfs/filer
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.filer.logs.type "existingClaim" }}
+        - name: seaweedfs-filer-log-volume
+          persistentVolumeClaim:
+            claimName: {{ .Values.filer.logs.claimName }}
+        {{- end }}
         {{- if eq .Values.filer.data.type "hostPath" }}
         - name: data-filer
           hostPath:
             path: {{ .Values.filer.data.hostPathPrefix }}/filer_store
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.filer.data.type "existingClaim" }}
+        - name: data-filer
+          persistentVolumeClaim:
+            claimName: {{ .Values.filer.data.claimName }}
+        {{- end }}
         - name: db-schema-config-volume
           configMap:
             name: seaweedfs-db-init-config
+        {{- if and .Values.filer.s3.enabled .Values.filer.s3.enableAuth }}
         - name: config-users
           secret:
             defaultMode: 420
+            {{- if .Values.filer.s3.existingConfigSecret }}
+            secretName: {{ .Values.filer.s3.existingConfigSecret }}
+            {{- else }}
             secretName: seaweedfs-s3-secret
+            {{- end }}
+        {{- end }}
         {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:
@@ -297,7 +326,7 @@ spec:
       nodeSelector:
         {{ tpl .Values.filer.nodeSelector . | indent 8 | trim }}
       {{- end }}
-  {{- if .Values.filer.enablePVC }}
+  {{- if and (.Values.filer.enablePVC) (eq .Values.filer.data.type "persistentVolumeClaim") }}
   # DEPRECATION: Deprecate in favor of filer.data section below
   volumeClaimTemplates:
   - metadata:
@@ -315,9 +344,13 @@ spec:
   {{- $pvc_exists := include "filer.pvc_exists" . -}}
   {{- if $pvc_exists }}
   volumeClaimTemplates:
-    {{- if eq .Values.filer.data.type "persistentVolumeClaim"}}
+    {{- if eq .Values.filer.data.type "persistentVolumeClaim" }}
     - metadata:
         name: data-filer
+        {{- with .Values.filer.data.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.filer.data.storageClass }}
@@ -325,9 +358,13 @@ spec:
           requests:
             storage: {{ .Values.filer.data.size }}
     {{- end }}
-    {{- if eq .Values.filer.logs.type "persistentVolumeClaim"}}
+    {{- if eq .Values.filer.logs.type "persistentVolumeClaim" }}
     - metadata:
         name: seaweedfs-filer-log-volume
+        {{- with .Values.filer.logs.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.filer.logs.storageClass }}

k8s/charts/seaweedfs/templates/filler-ingress.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.filer.enabled }}
 {{- if .Values.filer.ingress.enabled }}
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
 apiVersion: networking.k8s.io/v1
@@ -43,4 +44,5 @@ spec:
 {{- if .Values.filer.ingress.host }}
     host: {{ .Values.filer.ingress.host }}
 {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/master-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/master-ingress.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.master.enabled }}
 {{- if .Values.master.ingress.enabled }}
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
 apiVersion: networking.k8s.io/v1
@@ -22,6 +23,8 @@ metadata:
     app.kubernetes.io/component: master
 spec:
   ingressClassName: {{ .Values.master.ingress.className | quote }}
+  tls:
+    {{ .Values.master.ingress.tls | default list | toYaml | nindent 6 }}
   rules:
     - http:
         paths:
@@ -42,3 +45,4 @@ spec:
       host: {{ .Values.master.ingress.host }}
 {{- end }}
 {{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/master-service.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.master.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -9,7 +10,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
   annotations:
-    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"    
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
 spec:
   clusterIP: None
   publishNotReadyAddresses: true
@@ -31,3 +32,4 @@ spec:
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     app.kubernetes.io/component: master
+{{- end }}

k8s/charts/seaweedfs/templates/master-servicemonitor.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.master.enabled }}
 {{- if .Values.master.metricsPort }}
 {{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
@@ -18,7 +19,8 @@ spec:
       scrapeTimeout: 5s
   selector:
     matchLabels:
-      app: {{ template "seaweedfs.name" . }}
-      component: master
+      app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+      app.kubernetes.io/component: master
+{{- end }}
+{{- end }}
 {{- end }}
-{{- end }}

k8s/charts/seaweedfs/templates/master-statefulset.yaml

@@ -11,7 +11,7 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
 spec:
   serviceName: {{ template "seaweedfs.name" . }}-master
-  podManagementPolicy: Parallel
+  podManagementPolicy: {{ .Values.master.podManagementPolicy }}
   replicas: {{ .Values.master.replicas }}
   {{- if (gt (int .Values.master.updatePartition) 0) }}
   updateStrategy:
@@ -22,7 +22,6 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-      helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
       app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: master
   template:
@@ -32,6 +31,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: master
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.master.restartPolicy }}
       {{- if .Values.master.affinity }}
@@ -48,8 +54,8 @@ spec:
       priorityClassName: {{ .Values.master.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if .Values.master.serviceAccountName }}
-      serviceAccountName: {{ .Values.master.serviceAccountName | quote }}
+      {{- if .Values.global.createClusterRole }}
+      serviceAccountName: {{ .Values.master.serviceAccountName | default .Values.global.serviceAccountName | quote }} # for deleting statefulset pods after migration
       {{- end }}
       {{- if .Values.master.initContainers }}
       initContainers:
@@ -77,13 +83,23 @@ spec:
             {{- if .Values.master.extraEnvironmentVars }}
             {{- range $key, $value := .Values.master.extraEnvironmentVars }}
             - name: {{ $key }}
+            {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
             {{- end }}
             {{- end }}
             {{- if .Values.global.extraEnvironmentVars }}
             {{- range $key, $value := .Values.global.extraEnvironmentVars }}
             - name: {{ $key }}
+            {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
             {{- end }}
             {{- end }}
           command:
@@ -119,10 +135,11 @@ spec:
               {{- if .Values.master.metricsIntervalSec }}
               -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
               {{- end }}
-              {{- else if .Values.master.metricsPort }}
-              -metricsPort={{ .Values.master.metricsPort }} \
               {{- end }}
               {{- end }}
+              {{- if .Values.master.metricsPort }}
+              -metricsPort={{ .Values.master.metricsPort }} \
+              {{- end }}
               -volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \
               {{- if .Values.master.disableHttp }}
               -disableHttp \
@@ -205,8 +222,9 @@ spec:
           resources:
             {{ tpl .Values.master.resources . | nindent 12 | trim }}
           {{- end }}
-      {{- $hostpath_exists := include "master.hostpath_exists" . -}}
-      {{- if $hostpath_exists }}
+        {{- if .Values.master.sidecars }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
+        {{- end }}
       volumes:
         {{- if eq .Values.master.logs.type "hostPath" }}
         - name: seaweedfs-master-log-volume
@@ -214,12 +232,22 @@ spec:
             path: {{ .Values.master.logs.hostPathPrefix }}/logs/seaweedfs/master
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.master.logs.type "existingClaim" }}
+        - name: seaweedfs-master-log-volume
+          persistentVolumeClaim:
+            claimName: {{ .Values.master.logs.claimName }} 
+        {{- end }}
         {{- if eq .Values.master.data.type "hostPath" }}
         - name: data-{{ .Release.Namespace }}
           hostPath:
             path: {{ .Values.master.data.hostPathPrefix }}/seaweed-master/
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.master.data.type "existingClaim" }}
+        - name: data-{{ .Release.Namespace }}
+          persistentVolumeClaim:
+            claimName: {{ .Values.master.data.claimName }} 
+        {{- end }}
         - name: master-config
           configMap:
             name: {{ template "seaweedfs.name" . }}-master-config
@@ -244,7 +272,6 @@ spec:
             secretName: {{ template "seaweedfs.name" . }}-client-cert
         {{- end }}
         {{ tpl .Values.master.extraVolumes . | indent 8 | trim }}
-      {{- end }}
       {{- if .Values.master.nodeSelector }}
       nodeSelector:
         {{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
@@ -255,6 +282,10 @@ spec:
     {{- if eq .Values.master.data.type "persistentVolumeClaim"}}
     - metadata:
         name: data-{{ .Release.Namespace }}
+        {{- with .Values.master.data.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.master.data.storageClass }}
@@ -265,6 +296,10 @@ spec:
     {{- if eq .Values.master.logs.type "persistentVolumeClaim"}}
     - metadata:
         name: seaweedfs-master-log-volume
+        {{- with .Values.master.logs.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.master.logs.storageClass }}

k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml

@@ -0,0 +1,96 @@
+{{- if .Values.master.enabled }}
+{{- if .Values.filer.s3.enabled }}
+{{- if .Values.filer.s3.createBuckets }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ $.Release.Name }}-bucket-hook"
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+      labels:
+        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+        app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: post-install-job
+        image: {{ template "master.image" . }}
+        env:
+          - name: WEED_CLUSTER_DEFAULT
+            value: "sw"
+          - name: WEED_CLUSTER_SW_MASTER
+            value: "{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}:9333"
+          - name: WEED_CLUSTER_SW_FILER
+            value: "{{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:8888"
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: SEAWEEDFS_FULLNAME
+            value: "{{ template "seaweedfs.name" . }}"
+        command:
+          - "/bin/sh"
+          - "-ec"
+          - |
+          {{- range $reg, $props := $.Values.filer.s3.createBuckets }}
+            exec /bin/echo \
+            "s3.bucket.create --name {{ $props.name }}" |\
+            /usr/bin/weed shell
+          {{- end }}
+          {{- range $reg, $props := $.Values.filer.s3.createBuckets }}
+          {{- if $props.anonymousRead }}
+            exec /bin/echo \
+            "s3.configure --user anonymous \
+                --buckets {{ $props.name }} \
+                --actions Read \
+                --apply true" |\
+                /usr/bin/weed shell
+          {{- end }}  
+          {{- end }}
+        {{- if .Values.filer.s3.enableAuth }}
+        volumeMounts:
+          - name: config-users
+            mountPath: /etc/sw
+            readOnly: true
+        {{- end }}
+        ports:
+          - containerPort: {{ .Values.master.port }}
+            name: swfs-master
+          {{- if and .Values.global.monitoring.enabled .Values.master.metricsPort }}
+          - containerPort: {{ .Values.master.metricsPort }}
+            name: metrics
+          {{- end }}
+          - containerPort: {{ .Values.master.grpcPort }}
+            #name: swfs-master-grpc
+    {{- if .Values.filer.s3.enableAuth }}
+      volumes:
+        - name: config-users
+          secret:
+            defaultMode: 420
+            {{- if not (empty .Values.filer.s3.existingConfigSecret) }}
+            secretName: {{ .Values.filer.s3.existingConfigSecret }}
+            {{- else }}
+            secretName: seaweedfs-s3-secret
+            {{- end }}
+    {{- end }}{{/** if .Values.filer.s3.enableAuth **/}}
+{{- end }}{{/** if .Values.master.enabled **/}}
+{{- end }}{{/** if .Values.filer.s3.enabled **/}}
+{{- end }}{{/** if .Values.filer.s3.createBuckets **/}}

k8s/charts/seaweedfs/templates/s3-deployment.yaml

@@ -24,6 +24,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: s3
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.s3.restartPolicy }}
       {{- if .Values.s3.tolerations }}
@@ -62,16 +69,32 @@ spec:
                   fieldPath: metadata.namespace
             - name: SEAWEEDFS_FULLNAME
               value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.s3.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.s3.extraEnvironmentVars }}
+            - name: {{ $key }}
+            {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
+            {{- end }}
+            {{- end }}
             {{- if .Values.global.extraEnvironmentVars }}
             {{- range $key, $value := .Values.global.extraEnvironmentVars }}
             - name: {{ $key }}
+            {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
             {{- end }}
             {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
-            - | 
+            - |
               exec /usr/bin/weed \
                {{- if eq .Values.s3.logs.type "hostPath" }}
               -logdir=/logs \
@@ -114,9 +137,11 @@ spec:
             - name: logs
               mountPath: "/logs/"
             {{- end }}
+            {{- if .Values.s3.enableAuth }}
             - mountPath: /etc/sw
               name: config-users
               readOnly: true
+            {{- end }}
             {{- if .Values.global.enableSecurity }}
             - name: security-config
               readOnly: true
@@ -174,11 +199,20 @@ spec:
           resources:
             {{ tpl .Values.s3.resources . | nindent 12 | trim }}
           {{- end }}
+      {{- if .Values.s3.sidecars }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
+      {{- end }}
       volumes:
+        {{- if .Values.s3.enableAuth }}
         - name: config-users
           secret:
             defaultMode: 420
+            {{- if .Values.s3.existingConfigSecret }}
+            secretName: {{ .Values.s3.existingConfigSecret }}
+            {{- else }}
             secretName: seaweedfs-s3-secret
+            {{- end }}
+        {{- end }}
         {{- if eq .Values.s3.logs.type "hostPath" }}
         - name: logs
           hostPath:

k8s/charts/seaweedfs/templates/s3-ingress.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.s3.ingress.enabled }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1beta1
+{{- else }}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: ingress-{{ template "seaweedfs.name" . }}-s3
+  namespace: {{ .Release.Namespace }}
+  {{- with .Values.s3.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: s3
+spec:
+  ingressClassName: {{ .Values.s3.ingress.className | quote }}
+  tls:
+    {{ .Values.s3.ingress.tls | default list | toYaml | nindent 6}}
+  rules:
+  - http:
+      paths:
+      - path: /
+        pathType: ImplementationSpecific
+        backend:
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ template "seaweedfs.name" . }}-s3
+            port:
+              number: {{ .Values.s3.port }}
+              #name:
+{{- else }}
+          serviceName: {{ template "seaweedfs.name" . }}-s3
+          servicePort: {{ .Values.s3.port }}
+{{- end }}
+{{- if .Values.s3.ingress.host }}
+    host: {{ .Values.s3.ingress.host }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/s3-secret.yaml

@@ -1,4 +1,4 @@
-{{- if not (or .Values.filer.s3.skipAuthSecretCreation .Values.s3.skipAuthSecretCreation) }}
+{{- if or (and .Values.filer.s3.enabled .Values.filer.s3.enableAuth (not .Values.filer.s3.existingConfigSecret)) (and .Values.s3.enabled .Values.s3.enableAuth (not .Values.s3.existingConfigSecret)) }}
 {{- $access_key_admin := randAlphaNum 16 -}}
 {{- $secret_key_admin := randAlphaNum 32 -}}
 {{- $access_key_read := randAlphaNum 16 -}}
@@ -32,4 +32,4 @@ stringData:
   s3_auditLogConfig.json: |
     {{ toJson .Values.s3.auditLogConfig | nindent 4 }}
   {{- end }}
-{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/s3-service.yaml

@@ -1,3 +1,4 @@
+{{- if or .Values.s3.enabled .Values.filer.s3.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -9,6 +10,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
+  internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }}
   ports:
   - name: "swfs-s3"
     port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
@@ -29,3 +31,4 @@ spec:
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     app.kubernetes.io/component: {{ if .Values.s3.enabled }}s3{{ else }}filer{{ end }}
+{{- end }}

k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml

@@ -1,3 +1,4 @@
+{{- if or .Values.s3.enabled .Values.filer.s3.enabled }}
 {{- if .Values.s3.metricsPort }}
 {{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
@@ -21,4 +22,5 @@ spec:
       app: {{ template "seaweedfs.name" . }}
       component: s3
 {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/secret-seaweedfs-db.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.filer.enabled }}
 apiVersion: v1
 kind: Secret
 type: Opaque
@@ -17,3 +18,4 @@ stringData:
   password: "HardCodedPassword"
   # better to random generate and create in DB
   # password: {{ randAlphaNum 10 | sha256sum | b64enc | trunc 32 }}
+{{- end }}

k8s/charts/seaweedfs/templates/security-configmap.yaml

@@ -13,10 +13,38 @@ data:
   security.toml: |-
     # this file is read by master, volume server, and filer
 
+    {{- if .Values.global.securityConfig.jwtSigning.volumeWrite }}
     # the jwt signing key is read by master and volume server
     # a jwt expires in 10 seconds
     [jwt.signing]
     key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
+
+    {{- if .Values.global.securityConfig.jwtSigning.volumeRead }}
+    # this jwt signing key is read by master and volume server, and it is used for read operations:
+    # - the Master server generates the JWT, which can be used to read a certain file on a volume server
+    # - the Volume server validates the JWT on reading
+    [jwt.signing.read]
+    key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
+
+    {{- if .Values.global.securityConfig.jwtSigning.filerWrite }}
+    # If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
+    # - f.e. the S3 API Shim generates the JWT
+    # - the Filer server validates the JWT on writing
+    # the jwt defaults to expire after 10 seconds.
+    [jwt.filer_signing]
+    key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
+
+    {{- if .Values.global.securityConfig.jwtSigning.filerRead }}
+    # If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
+    # - f.e. the S3 API Shim generates the JWT
+    # - the Filer server validates the JWT on writing
+    # the jwt defaults to expire after 10 seconds.
+    [jwt.filer_signing.read]
+    key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
 
     # all grpc tls authentications are mutual
     # the values for the following ca, cert, and key are paths to the PERM files.

k8s/charts/seaweedfs/templates/service-account.yaml

@@ -4,7 +4,7 @@
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: seaweedfs-rw-cr
+  name: {{ .Values.global.serviceAccountName }}-rw-cr
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
@@ -18,7 +18,7 @@ rules:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: seaweedfs-rw-sa
+  name: {{ .Values.global.serviceAccountName }}
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
@@ -29,7 +29,7 @@ metadata:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: system:serviceaccount:seaweedfs-rw-sa:default
+  name: system:serviceaccount:{{ .Values.global.serviceAccountName }}:default
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
@@ -37,10 +37,10 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
 subjects:
 - kind: ServiceAccount
-  name: seaweedfs-rw-sa
+  name: {{ .Values.global.serviceAccountName }}
   namespace: {{ .Release.Namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: seaweedfs-rw-cr
+  name: {{ .Values.global.serviceAccountName }}-rw-cr
 {{- end }}

k8s/charts/seaweedfs/templates/volume-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/volume-service.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.volume.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -10,6 +11,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
   clusterIP: None
+  internalTrafficPolicy: {{ .Values.volume.internalTrafficPolicy | default "Cluster" }}
   ports:
   - name: "swfs-volume"
     port: {{ .Values.volume.port }}
@@ -27,4 +29,5 @@ spec:
 {{- end }}
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-    app.kubernetes.io/component: volume
+    app.kubernetes.io/component: volume
+{{- end }}

k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.volume.enabled }}
 {{- if .Values.volume.metricsPort }}
 {{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
@@ -21,4 +22,5 @@ spec:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
       app.kubernetes.io/component: volume
 {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/volume-statefulset.yaml

@@ -12,11 +12,10 @@ metadata:
 spec:
   serviceName: {{ template "seaweedfs.name" . }}-volume
   replicas: {{ .Values.volume.replicas }}
-  podManagementPolicy: Parallel
+  podManagementPolicy: {{ .Values.volume.podManagementPolicy }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-      helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
       app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: volume
   template:
@@ -26,6 +25,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: volume
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       {{- if .Values.volume.affinity }}
       affinity:
@@ -42,23 +48,25 @@ spec:
       priorityClassName: {{ .Values.volume.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if .Values.volume.serviceAccountName }}
-      serviceAccountName: {{ .Values.volume.serviceAccountName | quote }}
+      {{- if .Values.global.createClusterRole }}
+      serviceAccountName: {{ .Values.volume.serviceAccountName | default .Values.global.serviceAccountName | quote }} # for deleting statefulset pods after migration
       {{- end }}
       {{- $initContainers_exists := include "volume.initContainers_exists" . -}}
       {{- if $initContainers_exists }}
       initContainers:
-        {{- if .Values.volume.dir_idx }}
+        {{- if .Values.volume.idx }}
         - name: seaweedfs-vol-move-idx
           image: {{ template "volume.image" . }}
           imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }}
           command: [ '/bin/sh', '-c' ]
-          args: [ 'if ls {{ .Values.volume.dir }}/*.idx >/dev/null 2>&1; then mv {{ .Values.volume.dir }}/*.idx {{ .Values.volume.dir_idx }}/; fi;' ]
+          args: [ '{{range $dir :=  .Values.volume.dataDirs }}if ls /{{$dir.name}}/*.idx  >/dev/null 2>&1; then mv /{{$dir.name}}/*.idx /idx/ ; fi; {{end}}' ]
           volumeMounts:
             - name: idx
-              mountPath: {{ .Values.volume.dir_idx }}
-            - name: data
-              mountPath: {{ .Values.volume.dir }}
+              mountPath: /idx
+          {{- range $dir :=  .Values.volume.dataDirs }}
+            - name: {{ $dir.name }}
+              mountPath: /{{ $dir.name }}
+          {{- end }}
         {{- end }}
         {{- if .Values.volume.initContainers }}
         {{ tpl .Values.volume.initContainers . | nindent 8 | trim }}
@@ -83,10 +91,26 @@ spec:
                   fieldPath: status.hostIP
             - name: SEAWEEDFS_FULLNAME
               value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.volume.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.volume.extraEnvironmentVars }}
+            - name: {{ $key }}
+            {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
+            {{- end }}
+            {{- end }}
             {{- if .Values.global.extraEnvironmentVars }}
             {{- range $key, $value := .Values.global.extraEnvironmentVars }}
             - name: {{ $key }}
+            {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
             {{- end }}
             {{- end }}
           command:
@@ -94,7 +118,7 @@ spec:
             - "-ec"
             - |
               exec /usr/bin/weed \
-                {{- if eq .Values.volume.logs.type "hostPath" }}
+                {{- if .Values.volume.logs }}
                 -logdir=/logs \
                 {{- else }}
                 -logtostderr=true \
@@ -109,11 +133,11 @@ spec:
                 {{- if .Values.volume.metricsPort }}
                 -metricsPort={{ .Values.volume.metricsPort }} \
                 {{- end }}
-                -dir={{ .Values.volume.dir }} \
-                {{- if .Values.volume.dir_idx }}
-                -dir.idx={{ .Values.volume.dir_idx }} \
+                -dir {{range $index, $dir :=  .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}/{{$dir.name}}{{end}} \
+                {{- if .Values.volume.idx }}
+                -dir.idx=/idx \
                 {{- end }}
-                -max={{ .Values.volume.maxVolumes }} \
+                -max {{range $index, $dir :=  .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}{{$dir.maxVolumes}}{{end}} \
                 {{- if .Values.volume.rack }}
                 -rack={{ .Values.volume.rack }} \
                 {{- end }}
@@ -140,18 +164,20 @@ spec:
                 -minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
                 -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume.{{ .Release.Namespace }} \
                 -compactionMBps={{ .Values.volume.compactionMBps }} \
-                -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+                -mserver={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
           volumeMounts:
-            - name: data
-              mountPath: "{{ .Values.volume.dir }}/"
-            {{- if .Values.volume.dir_idx }}
-            - name: idx
-              mountPath: "{{ .Values.volume.dir_idx }}/"
+            {{- range $dir := .Values.volume.dataDirs }}
+            - name: {{ $dir.name }}
+              mountPath: "/{{ $dir.name }}/"
             {{- end }}
-            {{- if eq .Values.volume.logs.type "hostPath" }}
+            {{- if .Values.volume.logs }}
             - name: logs
               mountPath: "/logs/"
             {{- end }}
+            {{- if .Values.volume.idx }}
+            - name: idx
+              mountPath: "/idx/"
+            {{- end }}
             {{- if .Values.global.enableSecurity }}
             - name: security-config
               readOnly: true
@@ -178,7 +204,7 @@ spec:
             - containerPort: {{ .Values.volume.port }}
               name: swfs-vol
             {{- if .Values.volume.metricsPort }}
-            - containerPort: {{ .Values.filer.metricsPort }}
+            - containerPort: {{ .Values.volume.metricsPort }}
               name: metrics
             {{- end }}
             - containerPort: {{ .Values.volume.grpcPort }}
@@ -211,28 +237,55 @@ spec:
           resources:
             {{ tpl .Values.volume.resources . | nindent 12 | trim }}
           {{- end }}
-      {{- $hostpath_exists := include "volume.hostpath_exists" . -}}
-      {{- if $hostpath_exists }}
+      {{- if .Values.volume.sidecars }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }}
+      {{- end }}
       volumes:
-        {{- if eq .Values.volume.data.type "hostPath" }}
-        - name: data
+
+     {{- range $dir := .Values.volume.dataDirs }}
+
+      {{- if eq $dir.type "hostPath" }}
+        - name: {{ $dir.name }}
           hostPath:
-            path: {{ .Values.volume.data.hostPathPrefix }}/object_store/
+            path: {{ $dir.hostPathPrefix }}/object_store/
             type: DirectoryOrCreate
-        {{- end }}
-        {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
+      {{- end }}
+      {{- if eq $dir.type "existingClaim" }}
+        - name: {{ $dir.name }}
+          persistentVolumeClaim:
+            claimName: {{ $dir.claimName }}
+      {{- end }}
+
+     {{- end }}
+
+     {{- if .Values.volume.idx }}
+       {{- if eq .Values.volume.idx.type "hostPath" }}
         - name: idx
           hostPath:
             path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/
             type: DirectoryOrCreate
-        {{- end }}
-        {{- if eq .Values.volume.logs.type "hostPath" }}
+       {{- end }}
+       {{- if eq .Values.volume.idx.type "existingClaim" }}
+        - name: idx
+          persistentVolumeClaim:
+            claimName: {{ .Values.volume.idx.claimName }}
+       {{- end }}
+     {{- end }}
+
+     {{- if .Values.volume.logs }}
+       {{- if eq .Values.volume.logs.type "hostPath" }}
         - name: logs
           hostPath:
             path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume
             type: DirectoryOrCreate
-        {{- end }}
-        {{- if .Values.global.enableSecurity }}
+       {{- end }}
+       {{- if eq .Values.volume.logs.type "existingClaim" }}
+        - name: logs
+          persistentVolumeClaim:
+            claimName: {{ .Values.volume.logs.claimName }}
+       {{- end }}
+     {{- end }}
+     {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:
             name: {{ template "seaweedfs.name" . }}-security-config
@@ -251,31 +304,39 @@ spec:
         - name: client-cert
           secret:
             secretName: {{ template "seaweedfs.name" . }}-client-cert
-        {{- end }}
+      {{- end }}
       {{- if .Values.volume.extraVolumes }}
         {{ tpl .Values.volume.extraVolumes . | indent 8 | trim }}
       {{- end }}
-      {{- end }}
       {{- if .Values.volume.nodeSelector }}
       nodeSelector:
         {{ tpl .Values.volume.nodeSelector . | indent 8 | trim }}
       {{- end }}
-  {{- $pvc_exists := include "volume.pvc_exists" . -}}
-  {{- if $pvc_exists }}
   volumeClaimTemplates:
-    {{- if eq .Values.volume.data.type "persistentVolumeClaim"}}
+    {{- range $dir := .Values.volume.dataDirs }}
+    {{- if eq $dir.type "persistentVolumeClaim" }}
     - metadata:
-        name: data
+        name: {{ $dir.name }}
+        {{- with $dir.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
-        storageClassName: {{ .Values.volume.data.storageClass }}
+        storageClassName: {{ $dir.storageClass }}
         resources:
           requests:
-            storage: {{ .Values.volume.data.size }}
+            storage: {{ $dir.size }}
     {{- end }}
-    {{- if and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx }}
+    {{- end }}
+
+    {{- if and .Values.volume.idx (eq .Values.volume.idx.type "persistentVolumeClaim") }}
     - metadata:
         name: idx
+        {{- with .Values.volume.idx.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.volume.idx.storageClass }}
@@ -283,9 +344,13 @@ spec:
           requests:
             storage: {{ .Values.volume.idx.size }}
     {{- end }}
-    {{- if eq .Values.volume.logs.type "persistentVolumeClaim" }}
+    {{- if and .Values.volume.logs (eq .Values.volume.logs.type "persistentVolumeClaim") }}
     - metadata:
         name: logs
+        {{- with .Values.volume.logs.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.volume.logs.storageClass }}
@@ -293,5 +358,4 @@ spec:
           requests:
             storage: {{ .Values.volume.logs.size }}
     {{- end }}
-      {{- end }}
-{{- end }}
+    {{- end }}

k8s/charts/seaweedfs/values.yaml

@@ -10,6 +10,15 @@ global:
   restartPolicy: Always
   loggingLevel: 1
   enableSecurity: false
+  masterServer: null
+  securityConfig:
+    jwtSigning:
+      volumeWrite: true
+      volumeRead: false
+      filerWrite: false
+      filerRead: false
+  # we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings
+  serviceAccountName: "seaweedfs"
   certificates:
     alphacrds: false
   monitoring:
@@ -27,6 +36,10 @@ global:
     WEED_CLUSTER_DEFAULT: "sw"
     WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
     WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client.seaweedfs:8888"
+    # WEED_JWT_SIGNING_KEY:
+    #   secretKeyRef:
+    #     name: seaweedfs-signing-key
+    #     key: signingKey
 
 image:
   registry: ""
@@ -66,14 +79,21 @@ master:
     # Enter any extra configuration for master.toml here.
     # It may be be a multi-line string.
 
-  # can use ANY storage-class , example with local-path-provisioner
+  # You may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
   #  data:
   #    type: "persistentVolumeClaim"
   #    size: "24Ti"
   #    storageClass: "local-path-provisioner"
+  #    annotations:
+  #      "key": "value"
+  #
+  # You may also spacify an existing claim:
+  #  data:
+  #    type: "existingClaim"
+  #    claimName: "my-pvc"
   data:
     type: "hostPath"
-    size: ""
     storageClass: ""
     hostPathPrefix: /ssd
 
@@ -83,11 +103,25 @@ master:
     storageClass: ""
     hostPathPrefix: /storage
 
+  ## @param master.sidecars Add additional sidecar containers to the master pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  ## Set podManagementPolicy
+  podManagementPolicy: Parallel
+
   # Resource requests, limits, etc. for the master cluster placement. This
   # should map directly to the value of the resources field for a PodSpec,
   # formatted as a multi-line string. By default no direct resource request
@@ -120,7 +154,7 @@ master:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-    beta.kubernetes.io/arch: amd64
+    kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -153,12 +187,13 @@ master:
         sub_filter '=/' '=./';
         sub_filter '/seaweedfsstatic' './seaweedfsstatic';
         sub_filter_once off;
+    tls: []
 
   extraEnvironmentVars:
-    WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
-    WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
-    WEED_MASTER_VOLUME_GROWTH_COPY_3: 3
-    WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    WEED_MASTER_VOLUME_GROWTH_COPY_1: '7'
+    WEED_MASTER_VOLUME_GROWTH_COPY_2: '6'
+    WEED_MASTER_VOLUME_GROWTH_COPY_3: '3'
+    WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: '1'
 
   # used to configure livenessProbe on master-server containers
   #
@@ -208,40 +243,63 @@ volume:
   # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
   minFreeSpacePercent: 7
 
-# can use ANY storage-class , example with local-path-provisioner
-#  data:
-#    type: "persistentVolumeClaim"
-#    size: "24Ti"
-#    storageClass: "local-path-provisioner"
-  data:
-    type: "hostPath"
-    size: ""
-    storageClass: ""
-    hostPathPrefix: /storage
+  # For each data disk you may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
+  #  dataDirs:
+  #   - name: data:
+  #     type: "persistentVolumeClaim"
+  #     size: "24Ti"
+  #     storageClass: "local-path-provisioner"
+  #     annotations:
+  #      "key": "value"
+  #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
+  #
+  # You may also spacify an existing claim:
+  #    - name: data
+  #      type: "existingClaim"
+  #      claimName: "my-pvc"
+  #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
 
-  idx:
+  dataDirs:
+  - name: data1
     type: "hostPath"
-    size: ""
-    storageClass: ""
     hostPathPrefix: /ssd
+    maxVolumes: 0
 
-  logs:
-    type: "hostPath"
-    size: ""
-    storageClass: ""
-    hostPathPrefix: /storage
+   # - name: data2
+   #   type: "persistentVolumeClaim"
+   #   storageClass: "yourClassNameOfChoice"
+   #   size: "800Gi"
+   #   maxVolumes: 0
+
+  # idx can be defined by:
+  #
+  # idx:
+  #  type: "hostPath"
+  #  hostPathPrefix: /ssd
+  #
+  # or
+  #
+  # idx:
+  #  type: "persistentVolumeClaim"
+  #  size: "20Gi"
+  #  storageClass: "local-path-provisioner"
+  #
+  # or
+  #
+  # idx:
+  #   type: "existingClaim"
+  #   claimName: "myClaim"
+
+  # same applies to "logs"
+
+  idx: ""
+
+  logs: ""
 
   # limit background compaction or copying speed in mega bytes per second
   compactionMBps: "50"
 
-  # Directories to store data files. dir[,dir]... (default "/tmp")
-  dir: "/data"
-  # Directories to store index files. dir[,dir]... (default is the same as "dir")
-  dir_idx: null
-
-  # Maximum numbers of volumes, count[,count]...
-  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
-  maxVolumes: "0"
 
   # Volume server's rack name
   rack: null
@@ -258,11 +316,25 @@ volume:
   # Adjust jpg orientation when uploading.
   imagesFixOrientation: false
 
+  ## @param volume.sidecars Add additional sidecar containers to the volume pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  ## Set podManagementPolicy
+  podManagementPolicy: Parallel
+
   # Affinity Settings
   # Commenting out or setting as empty the affinity variable, will allow
   # deployment to single node services such as Minikube
@@ -280,7 +352,7 @@ volume:
   # should map directly to the value of the resources field for a PodSpec,
   # formatted as a multi-line string. By default no direct resource request
   # is made.
-  resources: {}
+  resources: null
 
   # Toleration Settings for server pods
   # This should be a multi-line string matching the Toleration array
@@ -291,7 +363,7 @@ volume:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-   beta.kubernetes.io/arch: amd64
+   kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-volume: "true"
 
@@ -303,6 +375,8 @@ volume:
   # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
   serviceAccountName: ""
 
+  extraEnvironmentVars:
+
   # used to configure livenessProbe on volume-server containers
   #
   livenessProbe:
@@ -373,12 +447,19 @@ filer:
   storage: 25Gi
   # storageClass is the class of storage which defaults to null (the Kube cluster will pick the default).
   storageClass: null
-
-  # can use ANY storage-class , example with local-path-provisioner
+  # You may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
   #  data:
   #    type: "persistentVolumeClaim"
   #    size: "24Ti"
   #    storageClass: "local-path-provisioner"
+  #    annotations:
+  #      "key": "value"
+  #
+  # You may also specify an existing claim:
+  #  data:
+  #    type: "existingClaim"
+  #    claimName: "my-pvc"
   data:
     type: "hostPath"
     size: ""
@@ -391,11 +472,25 @@ filer:
     storageClass: ""
     hostPathPrefix: /storage
 
+  ## @param filer.sidecars Add additional sidecar containers to the filer pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  ## Set podManagementPolicy
+  podManagementPolicy: Parallel
+
   # Affinity Settings
   # Commenting out or setting as empty the affinity variable, will allow
   # deployment to single node services such as Minikube
@@ -428,7 +523,7 @@ filer:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-    beta.kubernetes.io/arch: amd64
+    kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -521,7 +616,7 @@ filer:
       #     key: password
 
   s3:
-    enabled: true
+    enabled: false
     port: 8333
     # add additional https port
     httpsPort: 0
@@ -531,8 +626,17 @@ filer:
     domainName: ""
     # enable user & permission to s3 (need to inject to all services)
     enableAuth: false
-    skipAuthSecretCreation: false
+    # set to the name of an existing kubernetes Secret with the s3 json config file
+    # should have a secret key called seaweedfs_s3_config with an inline json configure
+    existingConfigSecret: null
     auditLogConfig: {}
+    # You may specify buckets to be created during the install process.
+    # Buckets may be exposed publicly by setting `anonymousRead` to `true`
+    # createBuckets:
+    #   - name: bucket-a
+    #     anonymousRead: true
+    #   - name: bucket-b
+    #     anonymousRead: false
 
 s3:
   enabled: false
@@ -551,12 +655,25 @@ s3:
   allowEmptyFolder: true
   # enable user & permission to s3 (need to inject to all services)
   enableAuth: false
-  skipAuthSecretCreation: false
+  # set to the name of an existing kubernetes Secret with the s3 json config file
+  # should have a secret key called seaweedfs_s3_config with an inline json config
+  existingConfigSecret: null
   auditLogConfig: {}
 
   # Suffix of the host name, {bucket}.{domainName}
   domainName: ""
 
+  ## @param s3.sidecars Add additional sidecar containers to the s3 pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
@@ -577,7 +694,7 @@ s3:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-    beta.kubernetes.io/arch: amd64
+    kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -595,6 +712,8 @@ s3:
     storageClass: ""
     hostPathPrefix: /storage
 
+  extraEnvironmentVars:
+
   # used to configure livenessProbe on s3 containers
   #
   livenessProbe:
@@ -621,6 +740,15 @@ s3:
     failureThreshold: 100
     timeoutSeconds: 10
 
+  ingress:
+    enabled: false
+    className: "nginx"
+    # host: false for "*" hostname
+    host: "seaweedfs.cluster.local"
+    # additional ingress annotations for the s3 endpoint
+    annotations: []
+    tls: []
+
 certificates:
   commonName: "SeaweedFS CA"
   ipAddresses: []
@@ -628,3 +756,13 @@ certificates:
   keySize: 2048
   duration: 2160h  # 90d
   renewBefore: 360h  # 15d
+  externalCertificates:
+    # This will avoid the need to use cert-manager and will rely on providing your own external certificates and CA
+    # you will need to store your provided certificates in the secret read by the different services:
+    # seaweedfs-master-cert, seaweedfs-filer-cert, etc. Can see any statefulset definition to see secret names
+    enabled: false
+
+# Labels to be added to all the created pods
+podLabels: {}
+# Annotations to be added to all the created pods
+podAnnotations: {}

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.55</version>
+    <version>3.59</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -16,8 +16,8 @@
     <properties>
         <protobuf.version>3.16.3</protobuf.version>
         <!-- follow https://github.com/grpc/grpc-java -->
-        <grpc.version>1.23.0</grpc.version>
-        <guava.version>30.0-jre</guava.version>
+        <grpc.version>1.53.0</grpc.version>
+        <guava.version>32.0.0-jre</guava.version>
     </properties>
 
     <dependencies>
@@ -96,6 +96,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.55</version>
+    <version>3.59</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -91,6 +91,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.55</version>
+    <version>3.59</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -90,6 +90,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -13,12 +13,12 @@ public class FilerClient extends FilerGrpcClient {
 
     private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
 
-    public FilerClient(String host, int grpcPort) {
-        super(host, grpcPort-10000, grpcPort);
+    public FilerClient(String filerHost, int filerGrpcPort) {
+        super(filerHost, filerGrpcPort-10000, filerGrpcPort);
     }
 
-    public FilerClient(String host, int port, int grpcPort) {
-        super(host, port, grpcPort);
+    public FilerClient(String filerHost, int filerPort, int filerGrpcPort) {
+        super(filerHost, filerPort, filerGrpcPort);
     }
 
     public static String toFileId(FilerProto.FileId fid) {

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -69,7 +69,7 @@ public class SeaweedRead {
             if (locations == null || locations.getLocationsCount() == 0) {
                 LOG.error("failed to locate {}", chunkView.fileId);
                 volumeIdCache.clearLocations(volumeId);
-                return 0;
+                throw new IOException("failed to locate fileId " + chunkView.fileId);
             }
 
             int len = readChunkView(filerClient, startOffset, buf, chunkView, locations);

other/java/client/src/main/proto/filer.proto

@@ -69,10 +69,13 @@ service SeaweedFiler {
     rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) {
     }
 
-    rpc Lock(LockRequest) returns (LockResponse) {
+    rpc DistributedLock(LockRequest) returns (LockResponse) {
     }
-    rpc Unlock(UnlockRequest) returns (UnlockResponse) {
+    rpc DistributedUnlock(UnlockRequest) returns (UnlockResponse) {
     }
+    rpc FindLockOwner(FindLockOwnerRequest) returns (FindLockOwnerResponse) {
+    }
+    // distributed lock management internal use only
     rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
     }
 }
@@ -421,6 +424,7 @@ message FilerConf {
         string data_center = 9;
         string rack = 10;
         string data_node = 11;
+        uint32 max_file_name_length = 12;
     }
     repeated PathConf locations = 2;
 }
@@ -444,6 +448,7 @@ message LockRequest {
     int64 seconds_to_lock = 2;
     string renew_token = 3;
     bool is_moved = 4;
+    string owner = 5;
 }
 message LockResponse {
     string renew_token = 1;
@@ -459,10 +464,18 @@ message UnlockResponse {
     string error = 1;
     string moved_to = 2;
 }
+message FindLockOwnerRequest {
+    string name = 1;
+    bool is_moved = 2;
+}
+message FindLockOwnerResponse {
+    string owner = 1;
+}
 message Lock {
     string name = 1;
     string renew_token = 2;
     int64 expired_at_ns = 3;
+    string owner = 4;
 }
 message TransferLocksRequest {
     repeated Lock locks = 1;

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>3.55</version>
+            <version>3.59</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>3.55</version>
+            <version>3.59</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/service/HFtpService.java

@@ -16,7 +16,7 @@ import org.springframework.stereotype.Component;
 import java.io.File;
 
 /**
- * reference: https://github.com/iponweb/hdfs-over-ftp
+ * reference: https://github.com/AShiou/hof
  */
 @Component
 public class HFtpService {

other/java/hdfs2/dependency-reduced-pom.xml

@@ -17,6 +17,7 @@
         <configuration>
           <source>8</source>
           <target>8</target>
+          <release>8</release>
         </configuration>
       </plugin>
       <plugin>
@@ -317,7 +318,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>3.55</seaweedfs.client.version>
+    <seaweedfs.client.version>3.59</seaweedfs.client.version>
     <hadoop.version>3.2.4</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>3.55</seaweedfs.client.version>
+        <seaweedfs.client.version>3.59</seaweedfs.client.version>
         <hadoop.version>3.2.4</hadoop.version>
     </properties>
 
@@ -33,6 +33,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/hdfs3/dependency-reduced-pom.xml

@@ -17,6 +17,7 @@
         <configuration>
           <source>8</source>
           <target>8</target>
+          <release>8</release>
         </configuration>
       </plugin>
       <plugin>
@@ -317,7 +318,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>3.55</seaweedfs.client.version>
+    <seaweedfs.client.version>3.59</seaweedfs.client.version>
     <hadoop.version>3.2.4</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>3.55</seaweedfs.client.version>
+        <seaweedfs.client.version>3.59</seaweedfs.client.version>
         <hadoop.version>3.2.4</hadoop.version>
     </properties>
 
@@ -33,6 +33,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

test/random_access/pom.xml

@@ -50,6 +50,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
         </plugins>

test/s3/compatibility/.gitignore

@@ -1,2 +1,8 @@
+# Test run scratch data
 /s3-tests
 /tmp
+
+# Test run outputs
+weed.log
+compat.raw.txt
+compat.summary.txt

test/s3/compatibility/Dockerfile

@@ -1,11 +1,21 @@
 # the tests only support python 3.6, not newer
-FROM ubuntu:latest
+#FROM ubuntu:latest
+FROM python:3.6.15-slim-buster
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
+# Installed required system deps
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
+
+# Install python deps
+RUN pip install virtualenv
+
+# Clone Ceph S3 tests
 RUN git clone https://github.com/ceph/s3-tests.git
+
 WORKDIR s3-tests
 
-# we pin a certain commit
+# Pin to a certain commit on ceph/s3-tests
+# https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
 RUN git checkout 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
 
 RUN ./bootstrap

test/s3/compatibility/Makefile

@@ -0,0 +1,9 @@
+.PHONY: all build-image test
+
+all: build-image test
+
+build-image:
+	./prepare.sh
+
+test:
+	./run.sh

test/s3/compatibility/README.md

@@ -1,13 +1,245 @@
 # Running S3 Compatibility tests against SeaweedFS
 
-This is using [the tests from CephFS](https://github.com/ceph/s3-tests).
+This is using [the tests from CephFS][s3-tests].
+
+[s3-tests]: https://github.com/ceph/s3-tests
 
 ## Prerequisites
 
-- have Docker installed
+- have [Docker][docker] installed
 - this has been executed on Mac. On Linux, the hostname in `s3tests.conf`  needs to be adjusted.
 
+[docker]: https://docs.docker.com
+
 ## Running tests
 
-- `./prepare.sh` to build the docker image
-- `./run.sh` to execute all tests
+To build the docker image that is used for the tests:
+
+```console
+./prepare.sh
+```
+
+To execute all tests:
+
+```console
+./run.sh
+```
+To see debug output including all commands run by the script:
+
+```console
+DEBUG=y ./run.sh
+```
+
+> [!WARNING]
+>
+> If your output does *not* look like the content in [`results.summary.txt`](./results.summary.txt)
+> and it is full of HTTP level exceptions, there is likely an error contacting the `weed` server from
+> the container that is runnin the S3 compatibility tests.
+>
+> There are at least a couple ways to solve this:
+>
+> - Modify your `docker` setup to ensure `host.docker.internal` is connected to your host running `weed`
+> - Use `--net=host` and modify `host` in `s3tests.conf` to `localhost`
+>
+> The `--net=host` solution is potentially *unsafe*, as the container running [s3-tests][s3-tests] could
+> visit unexpected websites or use the host-passthrough internet access maliciously.
+>
+> If you are OK with the risk of allowing `--net=host`:
+>
+> - Set `host = localhost` in `s3tests.conf`
+> - Set `DOCKER_NET_HOST=y` when running `run.sh`
+
+## Most recent results
+
+See [`results.summary.txt`](./results.summary.txt) for the latest results of compatibility testing (with the caveat that `s3-tests` is pinned to [`ceph/s3-tests` @ 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e](https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e)).
+
+The file is reproduced below for ease of access:
+
+```
+/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
+  warnings.warn(warning, PythonDeprecationWarning)
+s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
+s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
+s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
+s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
+s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
+s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
+s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
+s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
+s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
+s3tests_boto3.functional.test_s3.test_object_write_file ... ok
+s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
+s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
+s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
+s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
+s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
+s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
+s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
+s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
+s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
+s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
+s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
+s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
+ERROR
+
+======================================================================
+ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
+    self.test(*self.arg)
+  File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
+    'DisplayName': acl_response['Owner']['DisplayName'],
+KeyError: 'Owner'
+-------------------- >> begin captured logging << --------------------
+```

test/s3/compatibility/results.summary.txt

@@ -0,0 +1,186 @@
+/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
+  warnings.warn(warning, PythonDeprecationWarning)
+s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
+s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
+s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
+s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
+s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
+s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
+s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
+s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
+s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
+s3tests_boto3.functional.test_s3.test_object_write_file ... ok
+s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
+s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
+s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
+s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
+s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
+s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
+s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
+s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
+s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
+s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
+s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
+s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
+ERROR
+
+======================================================================
+ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
+    self.test(*self.arg)
+  File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
+    'DisplayName': acl_response['Owner']['DisplayName'],
+KeyError: 'Owner'
+-------------------- >> begin captured logging << --------------------

test/s3/compatibility/run.sh

@@ -1,24 +1,88 @@
 #!/usr/bin/env bash
 
-set -ex
+CONTAINER_NAME=${CONTAINER_NAME:-s3test-instance}
+CONF_FILE=${CONF_FILE:-s3tests.conf}
+WEED_BIN=${WEED_BIN:-../../../weed/weed}
+TEST_RAW_OUTPUT_FILE=${TEST_RAW_OUTPUT_FILE:-compat.raw.txt}
+TEST_PROCESSED_OUTPUT_FILE=${TEST_PROCESSED_OUTPUT_FILE:-compat.summary.txt}
 
+# Set up debugging for this bash script if DEBUG is set
+if [ -n "${DEBUG}" ]; then
+    echo -e "DEBUG set [${DEBUG}], enabling debugging output...";
+    set -ex
+fi
+
+# Reset from possible previous test run
 killall -9 weed || echo "already stopped"
 rm -Rf tmp
 mkdir tmp
-docker stop s3test-instance || echo "already stopped"
+docker stop $CONTAINER_NAME || echo "already stopped"
 
+# Ensure ulimit is set to reasonable value
 ulimit -n 10000
-../../../weed/weed server -filer -s3 -volume.max 0 -master.volumeSizeLimitMB 5 -dir "$(pwd)/tmp" 1>&2>weed.log &
 
+# Start weed w/ filer + s3 in the background
+$WEED_BIN server \
+          -filer \
+          -s3 \
+          -volume.max 0 \
+          -master.volumeSizeLimitMB 5 \
+          -dir "$(pwd)/tmp" \
+          1>&2>weed.log &
+
+# Wait for master to start up
+echo -e "\n[info] waiting for master @ 9333...";
 until curl --output /dev/null --silent --head --fail http://127.0.0.1:9333; do
-    printf '.'
-    sleep 5
+    printf '.';
+    sleep 5;
 done
-sleep 3
+sleep 3;
 
-rm -Rf logs-full.txt logs-summary.txt
-# docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py:test_get_obj_tagging -v  -a 'resource=object,!bucket-policy,!versioning,!encryption'
-docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v "$(pwd)"/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py -v  -a 'resource=object,!bucket-policy,!versioning,!encryption' | sed -n -e '/botocore.hooks/!p;//q' | tee logs-summary.txt
+# Wait for s3 to start up
+echo -e "\n[info] waiting for S3 @ 8333...";
+until curl --output /dev/null --silent --fail http://127.0.0.1:8333; do
+    printf '.';
+    sleep 5;
+done
+sleep 3;
 
-docker stop s3test-instance || echo "already stopped"
-killall -9 weed
+# Determine whether docker net
+DOCKER_NET_HOST_ARGS=""
+if [ -n "${DOCKER_NET_HOST}" ]; then
+    DOCKER_NET_HOST_ARGS="--net=host"
+    echo -e "\n[info] setting docker to het nost"
+fi
+
+echo -e "\n[warn] You may have to run with UNFILTERED=y to disable output filtering, if you get the broken pipe error";
+echo -e "\n[info] running tests with unfiltered output...";
+docker run \
+       --name $CONTAINER_NAME \
+       --rm \
+       ${DOCKER_NET_HOST_ARGS} \
+       -e S3TEST_CONF=$CONF_FILE \
+       -v "$(pwd)"/$CONF_FILE:/s3-tests/s3tests.conf \
+       -it \
+       s3tests \
+       ./virtualenv/bin/nosetests \
+       s3tests_boto3/functional/test_s3.py \
+       -v \
+       -a 'resource=object,!bucket-policy,!versioning,!encryption' \
+    | tee ${TEST_RAW_OUTPUT_FILE}
+
+# If the summary logs are present, process them
+if [ -f "${TEST_RAW_OUTPUT_FILE}" ]; then
+    cat ${TEST_RAW_OUTPUT_FILE} | sed -n -e '/botocore.hooks/!p;//q' | tee ${TEST_PROCESSED_OUTPUT_FILE}
+    echo -e "\n[info] ✅ Successfully wrote processed output @ [${TEST_PROCESSED_OUTPUT_FILE}]";
+    if [ -z "${TEST_KEEP_RAW_OUTPUT}" ]; then
+        echo -e "\n[info] removing test raw output file @ [${TEST_RAW_OUTPUT_FILE}] (to disable this, set TEST_KEEP_RAW_OUTPUT=y)...";
+        rm -rf ${TEST_RAW_OUTPUT_FILE};
+    fi
+else
+    echo -e "\n[warn] failed to find raw output @ [${TEST_RAW_OUTPUT_FILE}]";
+fi
+
+echo -e "\n[info] stopping [${CONTAINER_NAME}] container...";
+docker stop $CONTAINER_NAME || echo "[info] already stopped";
+
+echo -e "\n[info] stopping seaweedfs processes (all, via kill -9)...";
+killall -9 weed;

test/s3/compatibility/s3tests.conf

@@ -4,6 +4,14 @@
 # host set for rgw in vstart.sh
 host = host.docker.internal
 
+## NOTE: if running docker --net=host (e.x. `DOCKER_NET_HOST=y ./run.sh`)
+## then use 'localhost', so the container can more easily access your *local* weed instance
+##
+## This is *unsafe* -- it is possible that this script will do malicious
+## things with host-level access. only run this if you trust the programmatic content of
+## ceph/s3-tests (https://github.com/ceph/s3-tests) which will be using this file
+#host = localhost
+
 # port set for rgw in vstart.sh
 port = 8333
 

unmaintained/see_dat/see_dat.go

@@ -33,8 +33,8 @@ func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
 
 func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
 	t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
-	glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v",
-		*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t)
+	glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v name %s",
+		*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t, n.Name)
 	return nil
 }
 

weed/Makefile

@@ -7,7 +7,7 @@ all: install
 .PHONY : clean debug_mount
 
 install:
-	go install
+	go install -ldflags="-s -w"
 
 clean:
 	go clean $(SOURCE_DIR)
@@ -37,6 +37,10 @@ debug_s3:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 s3
 
+debug_mq:
+	go build -gcflags="all=-N -l"
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 mq.broker
+
 debug_filer_copy:
 	go build -gcflags="all=-N -l"
 	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- -v=4 filer.backup -filer=localhost:8888 -filerProxy -timeAgo=10h

weed/cluster/lock_client.go

@@ -16,13 +16,15 @@ type LockClient struct {
 	grpcDialOption  grpc.DialOption
 	maxLockDuration time.Duration
 	sleepDuration   time.Duration
+	seedFiler       pb.ServerAddress
 }
 
-func NewLockClient(grpcDialOption grpc.DialOption) *LockClient {
+func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *LockClient {
 	return &LockClient{
 		grpcDialOption:  grpcDialOption,
 		maxLockDuration: 5 * time.Second,
-		sleepDuration:   4 * time.Millisecond,
+		sleepDuration:   2473 * time.Millisecond,
+		seedFiler:       seedFiler,
 	}
 }
 
@@ -34,39 +36,78 @@ type LiveLock struct {
 	cancelCh       chan struct{}
 	grpcDialOption grpc.DialOption
 	isLocked       bool
-}
-
-// NewLockWithTimeout locks the key with the given duration
-func (lc *LockClient) NewLockWithTimeout(filer pb.ServerAddress, key string, lockDuration time.Duration) (lock *LiveLock) {
-	return lc.doNewLock(filer, key, lockDuration)
+	owner          string
+	lc             *LockClient
 }
 
 // NewLock creates a lock with a very long duration
-func (lc *LockClient) NewLock(filer pb.ServerAddress, key string) (lock *LiveLock) {
-	return lc.doNewLock(filer, key, lock_manager.MaxDuration)
+func (lc *LockClient) NewLock(key string, owner string) (lock *LiveLock) {
+	return lc.doNewLock(key, lock_manager.MaxDuration, owner)
 }
 
-func (lc *LockClient) doNewLock(filer pb.ServerAddress, key string, lockDuration time.Duration) (lock *LiveLock) {
+// StartLock starts a goroutine to lock the key and returns immediately.
+func (lc *LockClient) StartLock(key string, owner string) (lock *LiveLock) {
 	lock = &LiveLock{
 		key:            key,
-		filer:          filer,
+		filer:          lc.seedFiler,
+		cancelCh:       make(chan struct{}),
+		expireAtNs:     time.Now().Add(lock_manager.MaxDuration).UnixNano(),
+		grpcDialOption: lc.grpcDialOption,
+		owner:          owner,
+		lc:             lc,
+	}
+	go func() {
+		util.RetryUntil("create lock:"+key, func() error {
+			errorMessage, err := lock.doLock(lock_manager.MaxDuration)
+			if err != nil {
+				glog.V(0).Infof("create lock %s: %s", key, err)
+				time.Sleep(time.Second)
+				return err
+			}
+			if errorMessage != "" {
+				glog.V(4).Infof("create lock %s: %s", key, errorMessage)
+				time.Sleep(time.Second)
+				return fmt.Errorf("%v", errorMessage)
+			}
+			lock.isLocked = true
+			return nil
+		}, func(err error) (shouldContinue bool) {
+			if err != nil {
+				time.Sleep(time.Second)
+			}
+			return lock.renewToken == ""
+		})
+		lc.keepLock(lock)
+	}()
+	return
+}
+
+func (lc *LockClient) doNewLock(key string, lockDuration time.Duration, owner string) (lock *LiveLock) {
+	lock = &LiveLock{
+		key:            key,
+		filer:          lc.seedFiler,
 		cancelCh:       make(chan struct{}),
 		expireAtNs:     time.Now().Add(lockDuration).UnixNano(),
 		grpcDialOption: lc.grpcDialOption,
+		owner:          owner,
+		lc:             lc,
 	}
 	var needRenewal bool
 	if lockDuration > lc.maxLockDuration {
 		lockDuration = lc.maxLockDuration
 		needRenewal = true
 	}
-	util.RetryForever("create lock:"+key, func() error {
+	util.RetryUntil("create lock:"+key, func() error {
 		errorMessage, err := lock.doLock(lockDuration)
 		if err != nil {
+			time.Sleep(time.Second)
 			return err
 		}
 		if errorMessage != "" {
+			time.Sleep(time.Second)
 			return fmt.Errorf("%v", errorMessage)
 		}
+		lock.isLocked = true
 		return nil
 	}, func(err error) (shouldContinue bool) {
 		if err != nil {
@@ -75,8 +116,6 @@ func (lc *LockClient) doNewLock(filer pb.ServerAddress, key string, lockDuration
 		return lock.renewToken == ""
 	})
 
-	lock.isLocked = true
-
 	if needRenewal {
 		go lc.keepLock(lock)
 	}
@@ -88,10 +127,13 @@ func (lock *LiveLock) IsLocked() bool {
 	return lock.isLocked
 }
 
-func (lock *LiveLock) Unlock() error {
+func (lock *LiveLock) StopLock() error {
 	close(lock.cancelCh)
+	if !lock.isLocked {
+		return nil
+	}
 	return pb.WithFilerClient(false, 0, lock.filer, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-		_, err := client.Unlock(context.Background(), &filer_pb.UnlockRequest{
+		_, err := client.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
 			Name:       lock.key,
 			RenewToken: lock.renewToken,
 		})
@@ -100,11 +142,12 @@ func (lock *LiveLock) Unlock() error {
 }
 
 func (lc *LockClient) keepLock(lock *LiveLock) {
+	ticker := time.Tick(lc.sleepDuration)
 	for {
 		select {
-		case <-time.After(lc.sleepDuration):
+		case <-ticker:
 			// renew the lock if lock.expireAtNs is still greater than now
-			util.RetryForever("keep lock:"+lock.key, func() error {
+			util.RetryUntil("keep lock:"+lock.key, func() error {
 				lockDuration := time.Duration(lock.expireAtNs-time.Now().UnixNano()) * time.Nanosecond
 				if lockDuration > lc.maxLockDuration {
 					lockDuration = lc.maxLockDuration
@@ -116,10 +159,12 @@ func (lc *LockClient) keepLock(lock *LiveLock) {
 				errorMessage, err := lock.doLock(lockDuration)
 				if err != nil {
 					lock.isLocked = false
+					time.Sleep(time.Second)
 					return err
 				}
 				if errorMessage != "" {
 					lock.isLocked = false
+					time.Sleep(time.Second)
 					return fmt.Errorf("%v", errorMessage)
 				}
 				return nil
@@ -141,11 +186,12 @@ func (lc *LockClient) keepLock(lock *LiveLock) {
 
 func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, err error) {
 	err = pb.WithFilerClient(false, 0, lock.filer, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-		resp, err := client.Lock(context.Background(), &filer_pb.LockRequest{
+		resp, err := client.DistributedLock(context.Background(), &filer_pb.LockRequest{
 			Name:          lock.key,
 			SecondsToLock: int64(lockDuration.Seconds()),
 			RenewToken:    lock.renewToken,
 			IsMoved:       false,
+			Owner:         lock.owner,
 		})
 		if err == nil {
 			lock.renewToken = resp.RenewToken
@@ -154,6 +200,7 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
 			errorMessage = resp.Error
 			if resp.MovedTo != "" {
 				lock.filer = pb.ServerAddress(resp.MovedTo)
+				lock.lc.seedFiler = lock.filer
 			}
 		}
 		return err

weed/cluster/lock_manager/distributed_lock_manager.go

@@ -24,23 +24,38 @@ func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
 	}
 }
 
-func (dlm *DistributedLockManager) Lock(key string, token string) (renewToken string, movedTo pb.ServerAddress, err error) {
-	return dlm.LockWithTimeout(key, MaxDuration, token)
+func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (renewToken string, movedTo pb.ServerAddress, err error) {
+	movedTo, err = dlm.findLockOwningFiler(key)
+	if err != nil {
+		return
+	}
+	if movedTo != dlm.Host {
+		return
+	}
+	renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+	return
 }
 
-func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string) (renewToken string, movedTo pb.ServerAddress, err error) {
+func (dlm *DistributedLockManager) findLockOwningFiler(key string) (movedTo pb.ServerAddress, err error) {
 	servers := dlm.LockRing.GetSnapshot()
 	if servers == nil {
 		err = NoLockServerError
 		return
 	}
 
-	server := hashKeyToServer(key, servers)
-	if server != dlm.Host {
-		movedTo = server
+	movedTo = hashKeyToServer(key, servers)
+	return
+}
+
+func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, movedTo pb.ServerAddress, err error) {
+	movedTo, err = dlm.findLockOwningFiler(key)
+	if err != nil {
 		return
 	}
-	renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token)
+	if movedTo != dlm.Host {
+		return
+	}
+	owner, err = dlm.lockManager.GetLockOwner(key)
 	return
 }
 
@@ -62,8 +77,8 @@ func (dlm *DistributedLockManager) Unlock(key string, token string) (movedTo pb.
 
 // InsertLock is used to insert a lock to a server unconditionally
 // It is used when a server is down and the lock is moved to another server
-func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string) {
-	dlm.lockManager.InsertLock(key, expiredAtNs, token)
+func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string, owner string) {
+	dlm.lockManager.InsertLock(key, expiredAtNs, token, owner)
 }
 func (dlm *DistributedLockManager) SelectNotOwnedLocks(servers []pb.ServerAddress) (locks []*Lock) {
 	return dlm.lockManager.SelectLocks(func(key string) bool {

weed/cluster/lock_manager/lock_manager.go

@@ -12,7 +12,7 @@ var LockErrorNonEmptyTokenOnExpiredLock = fmt.Errorf("lock: non-empty token on a
 var LockErrorTokenMismatch = fmt.Errorf("lock: token mismatch")
 var UnlockErrorTokenMismatch = fmt.Errorf("unlock: token mismatch")
 
-// LockManager lock manager
+// LockManager local lock manager, used by distributed lock manager
 type LockManager struct {
 	locks *xsync.MapOf[string, *Lock]
 }
@@ -20,6 +20,7 @@ type Lock struct {
 	Token       string
 	ExpiredAtNs int64
 	Key         string // only used for moving locks
+	Owner       string
 }
 
 func NewLockManager() *LockManager {
@@ -30,7 +31,7 @@ func NewLockManager() *LockManager {
 	return t
 }
 
-func (lm *LockManager) Lock(path string, expiredAtNs int64, token string) (renewToken string, err error) {
+func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (renewToken string, err error) {
 	lm.locks.Compute(path, func(oldValue *Lock, loaded bool) (newValue *Lock, delete bool) {
 		if oldValue != nil {
 			if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < time.Now().UnixNano() {
@@ -41,14 +42,14 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string) (renew
 				} else {
 					// new lock
 					renewToken = uuid.New().String()
-					return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs}, false
+					return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}, false
 				}
 			}
 			// not expired
 			if oldValue.Token == token {
 				// token matches, renew the lock
 				renewToken = uuid.New().String()
-				return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs}, false
+				return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}, false
 			} else {
 				err = LockErrorTokenMismatch
 				return oldValue, false
@@ -57,7 +58,7 @@ func (lm *LockManager) Lock(path string, expiredAtNs int64, token string) (renew
 			if token == "" {
 				// new lock
 				renewToken = uuid.New().String()
-				return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs}, false
+				return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}, false
 			} else {
 				err = LockErrorNonEmptyTokenOnNewLock
 				return nil, false
@@ -132,6 +133,18 @@ func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Loc
 }
 
 // InsertLock inserts a lock unconditionally
-func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string) {
-	lm.locks.Store(path, &Lock{Token: token, ExpiredAtNs: expiredAtNs})
+func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string) {
+	lm.locks.Store(path, &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner})
+}
+
+func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
+	lm.locks.Range(func(k string, lock *Lock) bool {
+		if k == key && lock != nil {
+			owner = lock.Owner
+			return false
+		}
+		return true
+	})
+	return
+
 }

weed/command/filer.go

@@ -61,6 +61,7 @@ type FilerOptions struct {
 	showUIDirectoryDelete   *bool
 	downloadMaxMBps         *int
 	diskType                *string
+	allowedOrigins          *string
 }
 
 func init() {
@@ -91,6 +92,7 @@ func init() {
 	f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
 	f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
 	f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -98,6 +100,7 @@ func init() {
 	filerS3Options.portHttps = cmdFiler.Flag.Int("s3.port.https", 0, "s3 server https listen port")
 	filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
 	filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
+	filerS3Options.allowedOrigins = cmdFiler.Flag.String("s3.allowedOrigins", "*", "comma separated list of allowed origins")
 	filerS3Options.dataCenter = cmdFiler.Flag.String("s3.dataCenter", "", "prefer to read and write to volumes in this data center")
 	filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
@@ -169,7 +172,7 @@ func runFiler(cmd *Command, args []string) bool {
 
 	go stats_collect.StartMetricsServer(*f.bindIp, *f.metricsHttpPort)
 
-	filerAddress := util.JoinHostPort(*f.ip, *f.port)
+	filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
 	startDelay := time.Duration(2)
 	if *filerStartS3 {
 		filerS3Options.filer = &filerAddress
@@ -229,6 +232,9 @@ func (fo *FilerOptions) startFiler() {
 	if *fo.bindIp == "" {
 		*fo.bindIp = *fo.ip
 	}
+	if *fo.allowedOrigins == "" {
+		*fo.allowedOrigins = "*"
+	}
 
 	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
 
@@ -253,6 +259,7 @@ func (fo *FilerOptions) startFiler() {
 		ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
 		DownloadMaxBytesPs:    int64(*fo.downloadMaxMBps) * 1024 * 1024,
 		DiskType:              *fo.diskType,
+		AllowedOrigins:        strings.Split(*fo.allowedOrigins, ","),
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/filer_backup.go

@@ -8,6 +8,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"regexp"
 	"time"
 )
 
@@ -16,8 +17,10 @@ type FilerBackupOptions struct {
 	filer           *string
 	path            *string
 	excludePaths    *string
+	excludeFileName *string
 	debug           *bool
 	proxyByFiler    *bool
+	doDeleteFiles   *bool
 	timeAgo         *time.Duration
 	retentionDays   *int
 }
@@ -31,11 +34,12 @@ func init() {
 	filerBackupOptions.filer = cmdFilerBackup.Flag.String("filer", "localhost:8888", "filer of one SeaweedFS cluster")
 	filerBackupOptions.path = cmdFilerBackup.Flag.String("filerPath", "/", "directory to sync on filer")
 	filerBackupOptions.excludePaths = cmdFilerBackup.Flag.String("filerExcludePaths", "", "exclude directories to sync on filer")
+	filerBackupOptions.excludeFileName = cmdFilerBackup.Flag.String("filerExcludeFileName", "", "exclude file names that match the regexp to sync on filer")
 	filerBackupOptions.proxyByFiler = cmdFilerBackup.Flag.Bool("filerProxy", false, "read and write file chunks by filer instead of volume servers")
+	filerBackupOptions.doDeleteFiles = cmdFilerBackup.Flag.Bool("doDeleteFiles", false, "delete files on the destination")
 	filerBackupOptions.debug = cmdFilerBackup.Flag.Bool("debug", false, "debug mode to print out received files")
 	filerBackupOptions.timeAgo = cmdFilerBackup.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
 	filerBackupOptions.retentionDays = cmdFilerBackup.Flag.Int("retentionDays", 0, "incremental backup retention days")
-
 }
 
 var cmdFilerBackup = &Command{
@@ -90,6 +94,13 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 	sourceFiler := pb.ServerAddress(*backupOption.filer)
 	sourcePath := *backupOption.path
 	excludePaths := util.StringSplit(*backupOption.excludePaths, ",")
+	var reExcludeFileName *regexp.Regexp
+	if *backupOption.excludeFileName != "" {
+		var err error
+		if reExcludeFileName, err = regexp.Compile(*backupOption.excludeFileName); err != nil {
+			return fmt.Errorf("error compile regexp %v for exclude file name: %+v", *backupOption.excludeFileName, err)
+		}
+	}
 	timeAgo := *backupOption.timeAgo
 	targetPath := dataSink.GetSinkToDirectory()
 	debug := *backupOption.debug
@@ -119,7 +130,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 		*backupOption.proxyByFiler)
 	dataSink.SetSourceFiler(filerSource)
 
-	processEventFn := genProcessFunction(sourcePath, targetPath, excludePaths, dataSink, debug)
+	processEventFn := genProcessFunction(sourcePath, targetPath, excludePaths, reExcludeFileName, dataSink, *backupOption.doDeleteFiles, debug)
 
 	processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
 		glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3))

weed/command/filer_copy.go

@@ -31,21 +31,22 @@ var (
 )
 
 type CopyOptions struct {
-	include          *string
-	replication      *string
-	collection       *string
-	ttl              *string
-	diskType         *string
-	maxMB            *int
-	masterClient     *wdclient.MasterClient
-	concurrentFiles  *int
-	concurrentChunks *int
-	grpcDialOption   grpc.DialOption
-	masters          []string
-	cipher           bool
-	ttlSec           int32
-	checkSize        *bool
-	verbose          *bool
+	include            *string
+	replication        *string
+	collection         *string
+	ttl                *string
+	diskType           *string
+	maxMB              *int
+	masterClient       *wdclient.MasterClient
+	concurrentFiles    *int
+	concurrentChunks   *int
+	grpcDialOption     grpc.DialOption
+	masters            []string
+	cipher             bool
+	ttlSec             int32
+	checkSize          *bool
+	verbose            *bool
+	volumeServerAccess *string
 }
 
 func init() {
@@ -61,6 +62,7 @@ func init() {
 	copy.concurrentChunks = cmdFilerCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
 	copy.checkSize = cmdFilerCopy.Flag.Bool("check.size", false, "copy when the target file size is different from the source file")
 	copy.verbose = cmdFilerCopy.Flag.Bool("verbose", false, "print out details during copying")
+	copy.volumeServerAccess = cmdFilerCopy.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl]")
 }
 
 var cmdFilerCopy = &Command{
@@ -580,6 +582,9 @@ func (worker *FileCopyWorker) WithFilerClient(streamingMode bool, fn func(filer_
 }
 
 func (worker *FileCopyWorker) AdjustedUrl(location *filer_pb.Location) string {
+	if *worker.options.volumeServerAccess == "publicUrl" {
+		return location.PublicUrl
+	}
 	return location.Url
 }
 

weed/command/filer_remote_gateway.go

@@ -111,7 +111,7 @@ func runFilerRemoteGateway(cmd *Command, args []string) bool {
 
 	// synchronize /buckets folder
 	fmt.Printf("synchronize buckets in %s ...\n", remoteGatewayOptions.bucketsDir)
-	util.RetryForever("filer.remote.sync buckets", func() error {
+	util.RetryUntil("filer.remote.sync buckets", func() error {
 		return remoteGatewayOptions.followBucketUpdatesAndUploadToRemote(filerSource)
 	}, func(err error) bool {
 		if err != nil {

weed/command/filer_remote_gateway_buckets.go

@@ -30,13 +30,23 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
 		return err
 	}
 
-	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
-		lastTime := time.Unix(0, lastTsNs)
-		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
-		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, lastTsNs)
-	})
-
 	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo)
+	processor := NewMetadataProcessor(eachEntryFunc, 128, lastOffsetTs.UnixNano())
+
+	var lastLogTsNs = time.Now().UnixNano()
+	processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
+		processor.AddSyncJob(resp)
+		return nil
+	}, 3*time.Second, func(counter int64, lastTsNs int64) error {
+		offsetTsNs := processor.processedTsWatermark.Load()
+		if offsetTsNs == 0 {
+			return nil
+		}
+		now := time.Now().UnixNano()
+		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
+		lastLogTsNs = now
+		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, offsetTsNs)
+	})
 
 	option.clientEpoch++
 
@@ -224,7 +234,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 			if writeErr != nil {
 				return writeErr
 			}
-			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+			return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 		}
 		if filer_pb.IsDelete(resp) {
 			if resp.Directory == option.bucketsDir {
@@ -261,6 +271,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 					}
 				}
 			}
+			if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
+				return nil
+			}
 			oldBucket, oldRemoteStorageMountLocation, oldRemoteStorage, oldOk := option.detectBucketInfo(resp.Directory)
 			newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath)
 			if oldOk && newOk {
@@ -288,7 +301,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 						if writeErr != nil {
 							return writeErr
 						}
-						return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+						return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 					}
 				}
 			}
@@ -325,7 +338,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				if writeErr != nil {
 					return writeErr
 				}
-				return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+				return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 			}
 		}
 

weed/command/filer_remote_sync.go

@@ -90,7 +90,7 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
 
 	if dir != "" {
 		fmt.Printf("synchronize %s to remote storage...\n", dir)
-		util.RetryForever("filer.remote.sync "+dir, func() error {
+		util.RetryUntil("filer.remote.sync "+dir, func() error {
 			return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
 		}, func(err error) bool {
 			if err != nil {

weed/command/filer_remote_sync_dir.go

@@ -28,12 +28,13 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
 		return fmt.Errorf("read mount info: %v", detectErr)
 	}
 
-	eachEntryFunc, err := makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
+	eachEntryFunc, err := option.makeEventProcessor(remoteStorage, mountedDir, remoteStorageMountLocation, filerSource)
 	if err != nil {
 		return err
 	}
 
-	processor := NewMetadataProcessor(eachEntryFunc, 128)
+	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)
+	processor := NewMetadataProcessor(eachEntryFunc, 128, lastOffsetTs.UnixNano())
 
 	var lastLogTsNs = time.Now().UnixNano()
 	processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
@@ -50,18 +51,17 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
 		processor.AddSyncJob(resp)
 		return nil
 	}, 3*time.Second, func(counter int64, lastTsNs int64) error {
-		if processor.processedTsWatermark == 0 {
+		offsetTsNs := processor.processedTsWatermark.Load()
+		if offsetTsNs == 0 {
 			return nil
 		}
 		// use processor.processedTsWatermark instead of the lastTsNs from the most recent job
 		now := time.Now().UnixNano()
-		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, processor.processedTsWatermark), float64(counter)/(float64(now-lastLogTsNs)/1e9))
+		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
 		lastLogTsNs = now
-		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, processor.processedTsWatermark)
+		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, offsetTsNs)
 	})
 
-	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)
-
 	option.clientEpoch++
 
 	metadataFollowOption := &pb.MetadataFollowOption{
@@ -80,7 +80,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
 	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
 }
 
-func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
+func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
 	client, err := remote_storage.GetRemoteStorage(remoteStorage)
 	if err != nil {
 		return nil, err
@@ -152,7 +152,7 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 			if writeErr != nil {
 				return writeErr
 			}
-			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+			return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 		}
 		if filer_pb.IsDelete(resp) {
 			glog.V(2).Infof("delete: %+v", resp)
@@ -165,6 +165,9 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 			return client.DeleteFile(dest)
 		}
 		if message.OldEntry != nil && message.NewEntry != nil {
+			if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
+				return nil
+			}
 			oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
 			dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
 			if !shouldSendToRemote(message.NewEntry) {
@@ -191,7 +194,7 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 			if writeErr != nil {
 				return writeErr
 			}
-			return updateLocalEntry(&remoteSyncOptions, message.NewParentPath, message.NewEntry, remoteEntry)
+			return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry)
 		}
 
 		return nil
@@ -278,7 +281,10 @@ func updateLocalEntry(filerClient filer_pb.FilerClient, dir string, entry *filer
 }
 
 func isMultipartUploadFile(dir string, name string) bool {
+	return isMultipartUploadDir(dir) && strings.HasSuffix(name, ".part")
+}
+
+func isMultipartUploadDir(dir string) bool {
 	return strings.HasPrefix(dir, "/buckets/") &&
-		strings.Contains(dir, "/"+s3_constants.MultipartUploadsFolder+"/") &&
-		strings.HasSuffix(name, ".part")
+		strings.Contains(dir, "/"+s3_constants.MultipartUploadsFolder+"/")
 }

weed/command/filer_replication.go

@@ -99,7 +99,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 		if m.OldEntry != nil && m.NewEntry == nil {
 			glog.V(1).Infof("delete: %s", key)
 		} else if m.OldEntry == nil && m.NewEntry != nil {
-			glog.V(1).Infof("   add: %s", key)
+			glog.V(1).Infof("add: %s", key)
 		} else {
 			glog.V(1).Infof("modify: %s", key)
 		}

weed/command/filer_sync.go

@@ -17,7 +17,9 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util/grace"
 	"google.golang.org/grpc"
 	"os"
+	"regexp"
 	"strings"
+	"sync/atomic"
 	"time"
 )
 
@@ -46,8 +48,10 @@ type SyncOptions struct {
 	metricsHttpIp   *string
 	metricsHttpPort *int
 	concurrency     *int
+	aDoDeleteFiles  *bool
+	bDoDeleteFiles  *bool
 	clientId        int32
-	clientEpoch     int32
+	clientEpoch     atomic.Int32
 }
 
 const (
@@ -89,6 +93,8 @@ func init() {
 	syncMemProfile = cmdFilerSynchronize.Flag.String("memprofile", "", "memory profile output file")
 	syncOptions.metricsHttpIp = cmdFilerSynchronize.Flag.String("metricsIp", "", "metrics listen ip")
 	syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
+	syncOptions.aDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("a.doDeleteFiles", true, "delete and update files when synchronizing on filer A")
+	syncOptions.bDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("b.doDeleteFiles", true, "delete and update files when synchronizing on filer B")
 	syncOptions.clientId = util.RandomInt32()
 }
 
@@ -145,10 +151,10 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 			os.Exit(2)
 		}
 		for {
-			syncOptions.clientEpoch++
+			syncOptions.clientEpoch.Add(1)
 			err := doSubscribeFilerMetaChanges(
 				syncOptions.clientId,
-				syncOptions.clientEpoch,
+				syncOptions.clientEpoch.Load(),
 				grpcDialOption,
 				filerA,
 				*syncOptions.aPath,
@@ -163,6 +169,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 				*syncOptions.bDiskType,
 				*syncOptions.bDebug,
 				*syncOptions.concurrency,
+				*syncOptions.bDoDeleteFiles,
 				aFilerSignature,
 				bFilerSignature)
 			if err != nil {
@@ -182,10 +189,10 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 		}
 		go func() {
 			for {
-				syncOptions.clientEpoch++
+				syncOptions.clientEpoch.Add(1)
 				err := doSubscribeFilerMetaChanges(
 					syncOptions.clientId,
-					syncOptions.clientEpoch,
+					syncOptions.clientEpoch.Load(),
 					grpcDialOption,
 					filerB,
 					*syncOptions.bPath,
@@ -200,6 +207,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 					*syncOptions.aDiskType,
 					*syncOptions.aDebug,
 					*syncOptions.concurrency,
+					*syncOptions.aDoDeleteFiles,
 					bFilerSignature,
 					aFilerSignature)
 				if err != nil {
@@ -232,7 +240,7 @@ func initOffsetFromTsMs(grpcDialOption grpc.DialOption, targetFiler pb.ServerAdd
 }
 
 func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string,
-	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, sourceFilerSignature int32, targetFilerSignature int32) error {
+	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32) error {
 
 	// if first time, start from now
 	// if has previously synced, resume from that point of time
@@ -250,7 +258,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 	filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, grpcDialOption, sinkWriteChunkByFiler)
 	filerSink.SetSourceFiler(filerSource)
 
-	persistEventFn := genProcessFunction(sourcePath, targetPath, sourceExcludePaths, filerSink, debug)
+	persistEventFn := genProcessFunction(sourcePath, targetPath, sourceExcludePaths, nil, filerSink, doDeleteFiles, debug)
 
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
@@ -267,7 +275,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 		glog.Warningf("invalid concurrency value, using default: %d", DefaultConcurrencyLimit)
 		concurrency = DefaultConcurrencyLimit
 	}
-	processor := NewMetadataProcessor(processEventFn, concurrency)
+	processor := NewMetadataProcessor(processEventFn, concurrency, sourceFilerOffsetTsNs)
 
 	var lastLogTsNs = time.Now().UnixNano()
 	var clientName = fmt.Sprintf("syncFrom_%s_To_%s", string(sourceFiler), string(targetFiler))
@@ -275,16 +283,17 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
 		processor.AddSyncJob(resp)
 		return nil
 	}, 3*time.Second, func(counter int64, lastTsNs int64) error {
-		if processor.processedTsWatermark == 0 {
+		offsetTsNs := processor.processedTsWatermark.Load()
+		if offsetTsNs == 0 {
 			return nil
 		}
 		// use processor.processedTsWatermark instead of the lastTsNs from the most recent job
 		now := time.Now().UnixNano()
-		glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, processor.processedTsWatermark), float64(counter)/(float64(now-lastLogTsNs)/1e9))
+		glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
 		lastLogTsNs = now
 		// collect synchronous offset
-		statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(processor.processedTsWatermark))
-		return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, processor.processedTsWatermark)
+		statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(offsetTsNs))
+		return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
 	})
 
 	metadataFollowOption := &pb.MetadataFollowOption{
@@ -368,7 +377,7 @@ func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signature
 
 }
 
-func genProcessFunction(sourcePath string, targetPath string, excludePaths []string, dataSink sink.ReplicationSink, debug bool) func(resp *filer_pb.SubscribeMetadataResponse) error {
+func genProcessFunction(sourcePath string, targetPath string, excludePaths []string, reExcludeFileName *regexp.Regexp, dataSink sink.ReplicationSink, doDeleteFiles bool, debug bool) func(resp *filer_pb.SubscribeMetadataResponse) error {
 	// process function
 	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
 		message := resp.EventNotification
@@ -385,6 +394,10 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			glog.V(0).Infof("received %v", resp)
 		}
 
+		if isMultipartUploadDir(resp.Directory) {
+			return nil
+		}
+
 		if !strings.HasPrefix(resp.Directory, sourcePath) {
 			return nil
 		}
@@ -393,16 +406,22 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 				return nil
 			}
 		}
+		if reExcludeFileName != nil && reExcludeFileName.MatchString(message.NewEntry.Name) {
+			return nil
+		}
+		if dataSink.IsIncremental() {
+			doDeleteFiles = false
+		}
 		// handle deletions
 		if filer_pb.IsDelete(resp) {
+			if !doDeleteFiles {
+				return nil
+			}
 			if !strings.HasPrefix(string(sourceOldKey), sourcePath) {
 				return nil
 			}
 			key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
-			if !dataSink.IsIncremental() {
-				return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
-			}
-			return nil
+			return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
 		}
 
 		// handle new entries
@@ -428,7 +447,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			// old key is in the watched directory
 			if strings.HasPrefix(string(sourceNewKey), sourcePath) {
 				// new key is also in the watched directory
-				if !dataSink.IsIncremental() {
+				if doDeleteFiles {
 					oldKey := util.Join(targetPath, string(sourceOldKey)[len(sourcePath):])
 					message.NewParentPath = util.Join(targetPath, message.NewParentPath[len(sourcePath):])
 					foundExisting, err := dataSink.UpdateEntry(string(oldKey), message.OldEntry, message.NewParentPath, message.NewEntry, message.DeleteChunks, message.Signatures)
@@ -451,7 +470,7 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 
 			} else {
 				// new key is outside of the watched directory
-				if !dataSink.IsIncremental() {
+				if doDeleteFiles {
 					key := buildKey(dataSink, message, targetPath, sourceOldKey, sourcePath)
 					return dataSink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks, message.Signatures)
 				}

weed/command/filer_sync_jobs.go

@@ -6,6 +6,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sync"
+	"sync/atomic"
 )
 
 type MetadataProcessor struct {
@@ -14,15 +15,16 @@ type MetadataProcessor struct {
 	activeJobsCond       *sync.Cond
 	concurrencyLimit     int
 	fn                   pb.ProcessMetadataFunc
-	processedTsWatermark int64
+	processedTsWatermark atomic.Int64
 }
 
-func NewMetadataProcessor(fn pb.ProcessMetadataFunc, concurrency int) *MetadataProcessor {
+func NewMetadataProcessor(fn pb.ProcessMetadataFunc, concurrency int, offsetTsNs int64) *MetadataProcessor {
 	t := &MetadataProcessor{
 		fn:               fn,
 		activeJobs:       make(map[int64]*filer_pb.SubscribeMetadataResponse),
 		concurrencyLimit: concurrency,
 	}
+	t.processedTsWatermark.Store(offsetTsNs)
 	t.activeJobsCond = sync.NewCond(&t.activeJobsLock)
 	return t
 }
@@ -61,7 +63,7 @@ func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse)
 			}
 		}
 		if isOldest {
-			t.processedTsWatermark = resp.TsNs
+			t.processedTsWatermark.Store(resp.TsNs)
 		}
 		t.activeJobsCond.Signal()
 	}()