Compare commits

..
Author SHA1 Message Date
Chris Lu feef8ed40e refactor 2026-02-26 01:14:59 -08:00
Chris Lu 8e04f961f0 parameterize expected ec shard count 2026-02-26 01:01:10 -08:00
Chris Lu 5400fdca3a address comments 2026-02-26 00:58:29 -08:00
Chris Lu d3d4e354bf Create plan_test.go 2026-02-26 00:40:49 -08:00
Chris Lu df6f325574 ec repair: fix detection metadata 2026-02-26 00:31:58 -08:00
Chris Lu 0a6a76b71e test: share ec repair helpers 2026-02-26 00:31:53 -08:00
Chris LuGitHubgemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
93eff8fa45 Update weed/plugin/worker/ec_repair_handler.go
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
2026-02-26 00:26:40 -08:00
Chris Lu cc53bb17ca ci: run ec repair plugin worker tests 2026-02-26 00:01:15 -08:00
Chris Lu e1f2e7e357 test: add ec repair plugin worker coverage 2026-02-26 00:01:08 -08:00
Chris Lu 6e715f14d5 test: extend fake volume server for ec repair 2026-02-26 00:01:02 -08:00
Chris Lu b09f4cc3e7 ec repair: add plugin worker handler 2026-02-26 00:00:57 -08:00
Chris Lu a8fab5503c ec repair: add shard analysis helpers 2026-02-26 00:00:52 -08:00
385 changed files with 7037 additions and 31921 deletions
+5 -5
View File
@@ -32,7 +32,7 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
-
name: Create BuildKit config
run: |
@@ -42,28 +42,28 @@ jobs:
EOF
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
-
name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
-
name: Build
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
@@ -126,14 +126,14 @@ jobs:
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
@@ -150,7 +150,7 @@ jobs:
fi
- name: Build and push image
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+7 -7
View File
@@ -98,7 +98,7 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
- name: Set up QEMU
if: matrix.platform != 'amd64'
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
run: |
cat > /tmp/buildkitd.toml <<EOF
@@ -106,25 +106,25 @@ jobs:
mirrors = ["https://mirror.gcr.io"]
EOF
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
with:
buildkitd-flags: "--debug"
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }}
- name: Build ${{ matrix.platform }} ${{ matrix.variant }}
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
with:
@@ -180,12 +180,12 @@ jobs:
ghcr.io/chrislusf/seaweedfs
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
- name: Login to Docker Hub
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
@@ -35,14 +35,14 @@ jobs:
org.opencontainers.image.vendor=Chris Lu
-
name: Set up QEMU
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
@@ -57,7 +57,7 @@ jobs:
fi
-
name: Build
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
with:
context: ./docker
push: ${{ github.event_name != 'pull_request' }}
+12 -12
View File
@@ -60,16 +60,16 @@ jobs:
build_args: TAGS=5BytesOffset
tag_suffix: _large_disk
# Full tags - multi-arch
# Full tags - amd64 only
- variant: full
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _full
# Large disk + full tags - multi-arch
# Large disk + full tags - amd64 only
- variant: large_disk_full
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _large_disk_full
@@ -117,7 +117,7 @@ jobs:
- name: Set up QEMU
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@v3
- name: Create BuildKit config
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
@@ -129,20 +129,20 @@ jobs:
- name: Set up Docker Buildx
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
with:
buildkitd-config: /tmp/buildkitd.toml
- name: Login to Docker Hub
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
@@ -150,7 +150,7 @@ jobs:
- name: Build and push ${{ matrix.variant }}
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/build-push-action@v7
uses: docker/build-push-action@v6
env:
DOCKER_BUILDKIT: 1
with:
@@ -198,14 +198,14 @@ jobs:
steps:
- name: Login to Docker Hub
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GHCR
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }}
@@ -82,19 +82,19 @@ jobs:
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
- name: Set up QEMU
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
- name: Login to Docker Hub
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push image
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
with:
context: ./docker
push: true
+1 -1
View File
@@ -26,7 +26,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: '1.24'
- name: Build Telemetry Server
if: github.event_name == 'workflow_dispatch' && inputs.deploy
+1 -1
View File
@@ -11,4 +11,4 @@ jobs:
- name: 'Checkout Repository'
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
- name: 'Dependency Review'
uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48
uses: actions/dependency-review-action@05fe4576374b728f0c523d6a13d64c25081e0803
+9 -8
View File
@@ -23,16 +23,17 @@ jobs:
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
- name: Set up Go 1.x
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
with:
go-version-file: 'go.mod'
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Cache Docker layers
uses: actions/cache@v5
@@ -134,7 +135,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: output-logs
path: docker/output.log
+2 -2
View File
@@ -18,7 +18,7 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
id: go
- name: Check out code into the Go module directory
@@ -52,7 +52,7 @@ jobs:
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ec-integration-test-logs
path: |
+2 -2
View File
@@ -28,7 +28,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: '1.24'
- name: Build weed binary
run: |
@@ -42,7 +42,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ec-test-logs
path: test/erasure_coding/admin_dockertest/tmp/logs/
+4 -3
View File
@@ -22,6 +22,7 @@ permissions:
contents: read
env:
GO_VERSION: '1.24'
TEST_TIMEOUT: '45m'
jobs:
@@ -34,10 +35,10 @@ jobs:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: ${{ env.GO_VERSION }}
- name: Install FUSE and dependencies
run: |
@@ -182,7 +183,7 @@ jobs:
- name: Upload Test Artifacts
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: fuse-integration-test-results
path: |
+18 -15
View File
@@ -19,12 +19,13 @@ jobs:
name: Go Vet
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
- name: Set up Go 1.x
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
with:
go-version-file: 'go.mod'
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Get dependencies
run: |
cd weed; go get -v -t -d ./...
@@ -41,12 +42,13 @@ jobs:
name: Build
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
- name: Set up Go 1.x
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
with:
go-version-file: 'go.mod'
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Build
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
@@ -54,11 +56,12 @@ jobs:
name: Test
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
- name: Set up Go 1.x
uses: actions/setup-go@a5f9b05d2d216f63e13859e0d847461041025775 # v2
with:
go-version-file: 'go.mod'
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
- name: Test
run: cd weed; go test -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v ./...
+1 -1
View File
@@ -49,7 +49,7 @@ jobs:
- name: Upload Test Reports
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: test-reports-java-${{ matrix.java }}
path: |
+2 -2
View File
@@ -26,14 +26,14 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
cache-dependency-path: |
**/go.sum
id: go
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Install dependencies
run: |
+7 -7
View File
@@ -37,7 +37,7 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
id: go
- name: Check out code
@@ -82,7 +82,7 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
id: go
- name: Check out code
@@ -132,7 +132,7 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
cache-dependency-path: |
**/go.sum
@@ -311,7 +311,7 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
cache-dependency-path: |
**/go.sum
@@ -473,7 +473,7 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
cache-dependency-path: |
**/go.sum
@@ -631,7 +631,7 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
cache-dependency-path: |
**/go.sum
@@ -789,7 +789,7 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
id: go
- name: Check out code
@@ -30,6 +30,7 @@ permissions:
contents: read
env:
GO_VERSION: '1.24'
TEST_TIMEOUT: '10m'
jobs:
@@ -42,10 +43,10 @@ jobs:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: ${{ env.GO_VERSION }}
- name: Build SeaweedFS
run: |
@@ -69,7 +70,7 @@ jobs:
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: metadata-subscribe-test-logs
path: |
+2
View File
@@ -20,6 +20,8 @@ jobs:
include:
- worker: erasure_coding
path: test/plugin_workers/erasure_coding
- worker: ec_repair
path: test/plugin_workers/ec_repair
- worker: vacuum
path: test/plugin_workers/vacuum
- worker: volume_balance
+3 -3
View File
@@ -25,14 +25,14 @@ jobs:
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
id: go
- name: Check out code
uses: actions/checkout@v6
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Cache Docker layers
uses: actions/cache@v5
@@ -62,7 +62,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: postgres-logs
path: test/postgres/postgres-output.log
@@ -57,7 +57,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: integration-test-logs
path: test/s3/normal/*.log
+1 -1
View File
@@ -77,7 +77,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-filer-group-test-logs
path: test/s3/filer_group/weed-test*.log
+9 -9
View File
@@ -76,7 +76,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-test-logs-${{ matrix.test-type }}
path: test/s3/versioning/weed-test*.log
@@ -124,7 +124,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-compatibility-logs
path: test/s3/versioning/weed-test*.log
@@ -172,7 +172,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-cors-compatibility-logs
path: test/s3/cors/weed-test*.log
@@ -239,7 +239,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-retention-test-logs-${{ matrix.test-type }}
path: test/s3/retention/weed-test*.log
@@ -306,7 +306,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-cors-test-logs-${{ matrix.test-type }}
path: test/s3/cors/weed-test*.log
@@ -355,7 +355,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-retention-worm-logs
path: test/s3/retention/weed-test*.log
@@ -422,7 +422,7 @@ jobs:
- name: Upload stress test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-stress-logs
path: test/s3/versioning/weed-test*.log
@@ -478,7 +478,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-tagging-test-logs
path: test/s3/tagging/weed-test*.log
@@ -531,7 +531,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-remote-cache-test-logs
path: |
+6 -15
View File
@@ -5,8 +5,6 @@ on:
paths:
- 'weed/iam/**'
- 'weed/s3api/**'
- 'weed/credential/**'
- 'weed/pb/**'
- 'test/s3/iam/**'
- '.github/workflows/s3-iam-tests.yml'
push:
@@ -14,8 +12,6 @@ on:
paths:
- 'weed/iam/**'
- 'weed/s3api/**'
- 'weed/credential/**'
- 'weed/pb/**'
- 'test/s3/iam/**'
- '.github/workflows/s3-iam-tests.yml'
@@ -69,7 +65,7 @@ jobs:
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: iam-unit-test-results
path: |
@@ -84,7 +80,7 @@ jobs:
timeout-minutes: 25
strategy:
matrix:
test-type: ["basic", "advanced", "policy-enforcement", "group"]
test-type: ["basic", "advanced", "policy-enforcement"]
steps:
- name: Check out code
@@ -121,7 +117,7 @@ jobs:
"basic")
echo "Running basic IAM functionality tests..."
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAMUserManagement|TestIAMAccessKeyManagement|TestIAMPolicyManagement" ./...
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAM" ./...
;;
"advanced")
echo "Running advanced IAM feature tests..."
@@ -133,11 +129,6 @@ jobs:
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestS3IAMPolicyEnforcement|TestS3IAMBucketPolicy|TestS3IAMContextual" ./...
;;
"group")
echo "Running IAM group management tests..."
make clean setup start-services wait-for-services
go test -v -timeout 15m -run "TestIAMGroup" ./...
;;
*)
echo "Unknown test type: ${{ matrix.test-type }}"
exit 1
@@ -171,7 +162,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-integration-logs-${{ matrix.test-type }}
path: test/s3/iam/weed-*.log
@@ -231,7 +222,7 @@ jobs:
- name: Upload distributed test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-distributed-logs
path: test/s3/iam/weed-*.log
@@ -283,7 +274,7 @@ jobs:
- name: Upload performance test results
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-performance-results
path: |
+1 -1
View File
@@ -152,7 +152,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-keycloak-test-logs
path: |
+3 -3
View File
@@ -41,7 +41,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
- name: Set up Python ${{ matrix.python-version }}
@@ -121,7 +121,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: test-logs-python-${{ matrix.python-version }}
path: |
@@ -148,7 +148,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
cache: true
- name: Run Go unit tests
+4 -4
View File
@@ -70,7 +70,7 @@ jobs:
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: policy-unit-test-results
path: |
@@ -178,7 +178,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-policy-variables-test-logs
path: /tmp/weed_policy_test_server.log
@@ -299,7 +299,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
@@ -386,7 +386,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: trusted-proxy-test-logs
path: /tmp/weed_proxy_test.log
@@ -1,122 +0,0 @@
name: "S3 Proxy Signature Tests"
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
concurrency:
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
cancel-in-progress: true
permissions:
contents: read
jobs:
proxy-signature-tests:
name: S3 Proxy Signature Verification Tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v6
- name: Set up Go 1.x
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
- name: Build SeaweedFS binary for Linux
run: |
set -x
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs=false -v -o test/s3/proxy_signature/weed ./weed
- name: Run S3 Proxy Signature Tests
timeout-minutes: 10
working-directory: test/s3/proxy_signature
run: |
set -x
echo "Starting Docker Compose services..."
docker compose up -d --build
# Check if containers are running
echo "Checking container status..."
docker compose ps
# Wait for services to be ready
echo "Waiting for nginx proxy to be ready..."
PROXY_READY=0
for i in $(seq 1 30); do
if curl -s http://localhost:9000/ > /dev/null 2>&1; then
echo "Proxy is ready"
PROXY_READY=1
break
fi
echo "Waiting for proxy... ($i/30)"
sleep 1
done
if [ $PROXY_READY -eq 0 ]; then
echo "ERROR: Proxy failed to become ready after 30 seconds"
echo "Docker compose logs:"
docker compose logs --no-color || true
exit 1
fi
# Wait for SeaweedFS to be ready
echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
S3_READY=0
for i in $(seq 1 30); do
# Check logs first for startup message (weed mini says "S3 service is ready")
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)"; then
echo "SeaweedFS S3 gateway is ready"
S3_READY=1
break
fi
# Fallback: check headers via proxy (which is already ready)
if curl -s -I http://localhost:9000/ | grep -qi "SeaweedFS"; then
echo "SeaweedFS S3 gateway is responding via proxy"
S3_READY=1
break
fi
echo "Waiting for S3 gateway... ($i/30)"
sleep 1
done
if [ $S3_READY -eq 0 ]; then
echo "ERROR: SeaweedFS S3 gateway failed to become ready after 30 seconds"
echo "Latest seaweedfs logs:"
docker compose logs --no-color --tail 20 seaweedfs || true
exit 1
fi
# Run the test script inside AWS CLI container
echo "Running test script..."
docker run --rm --network host \
--entrypoint bash \
amazon/aws-cli:latest \
-c "$(cat test.sh)"
TEST_RESULT=$?
# Cleanup
docker compose down
exit $TEST_RESULT
- name: Cleanup on failure
if: failure()
working-directory: test/s3/proxy_signature
run: |
echo "Cleaning up Docker containers..."
ls -al weed || true
ldd weed || true
echo "Docker compose logs:"
docker compose logs --no-color || true
echo "Container status before cleanup:"
docker ps -a
echo "Stopping services..."
docker compose down || true
+2 -2
View File
@@ -33,7 +33,7 @@ jobs:
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Install SeaweedFS
run: |
@@ -73,7 +73,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-spark-test-logs
path: test/s3/spark/test-output.log
+7 -7
View File
@@ -95,7 +95,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-test-logs-${{ matrix.test-type }}
path: test/s3/sse/weed-test*.log
@@ -143,7 +143,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-compatibility-logs
path: test/s3/sse/weed-test*.log
@@ -192,7 +192,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-metadata-persistence-logs
path: test/s3/sse/weed-test*.log
@@ -241,7 +241,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-copy-operations-logs
path: test/s3/sse/weed-test*.log
@@ -290,7 +290,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-multipart-logs
path: test/s3/sse/weed-test*.log
@@ -340,7 +340,7 @@ jobs:
- name: Upload performance test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-performance-logs
path: test/s3/sse/weed-test*.log
@@ -389,7 +389,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-volume-encryption-logs
path: /tmp/seaweedfs-sse-*.log
+12 -73
View File
@@ -66,7 +66,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-tables-test-logs
path: test/s3tables/table-buckets/test-output.log
@@ -122,7 +122,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: iceberg-catalog-test-logs
path: test/s3tables/catalog/test-output.log
@@ -144,7 +144,7 @@ jobs:
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Pre-pull Trino image
run: docker pull trinodb/trino:479
@@ -188,73 +188,12 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: trino-iceberg-catalog-test-logs
path: test/s3tables/catalog_trino/test-output.log
retention-days: 3
polaris-integration-tests:
name: Polaris Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Pre-pull Polaris image
run: docker pull apache/polaris:latest
- name: Run Polaris Integration Tests
timeout-minutes: 25
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Polaris Tests ==="
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
echo "Polaris integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/polaris
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: polaris-test-logs
path: test/s3tables/polaris/test-output.log
retention-days: 3
spark-iceberg-catalog-tests:
name: Spark Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
@@ -271,7 +210,7 @@ jobs:
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Pre-pull Spark image
run: docker pull apache/spark:3.5.1
@@ -315,7 +254,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: spark-iceberg-catalog-test-logs
path: test/s3tables/catalog_spark/test-output.log
@@ -337,7 +276,7 @@ jobs:
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Pre-pull RisingWave image
run: |
@@ -383,7 +322,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: risingwave-catalog-test-logs
path: test/s3tables/catalog_risingwave/test-output.log
@@ -405,7 +344,7 @@ jobs:
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Pre-pull Python image
run: docker pull python:3
@@ -449,7 +388,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: sts-integration-test-logs
path: test/s3tables/sts_integration/test-output.log
@@ -471,7 +410,7 @@ jobs:
id: go
- name: Set up Docker
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@v3
- name: Pre-pull Python image
run: docker pull python:3
@@ -518,7 +457,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: lakekeeper-integration-test-logs
path: test/s3tables/lakekeeper/test-output.log
+3 -2
View File
@@ -24,6 +24,7 @@ permissions:
contents: read
env:
GO_VERSION: '1.24'
TEST_TIMEOUT: '15m'
jobs:
@@ -36,10 +37,10 @@ jobs:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: ${{ env.GO_VERSION }}
- name: Install dependencies
run: |
@@ -43,7 +43,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: '1.24'
- name: Build SeaweedFS binary
run: |
@@ -125,7 +125,7 @@ jobs:
- name: Upload test results
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: spark-test-results
path: test/java/spark/target/surefire-reports/
@@ -24,7 +24,7 @@ jobs:
- uses: actions/setup-go@v6
with:
go-version: ^1.25
go-version: ^1.24
- name: Build SeaweedFS
run: |
@@ -103,7 +103,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: seaweedfs-logs
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
+1 -1
View File
@@ -106,7 +106,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: tus-test-logs
path: |
@@ -28,6 +28,7 @@ permissions:
contents: read
env:
GO_VERSION: '1.24'
TEST_TIMEOUT: '30m'
jobs:
@@ -45,10 +46,10 @@ jobs:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
- name: Set up Go ${{ env.GO_VERSION }}
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
go-version: ${{ env.GO_VERSION }}
- name: Build SeaweedFS binary
run: |
@@ -89,7 +90,7 @@ jobs:
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: volume-server-integration-test-logs
path: /tmp/volume-server-it-logs/
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.25 AS builder
FROM golang:1.24 AS builder
RUN apt-get update && \
apt-get install -y build-essential wget ca-certificates && \
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.25-alpine AS builder
FROM golang:1.24-alpine as builder
RUN apk add git g++ fuse
RUN mkdir -p /go/src/github.com/seaweedfs/
ARG BRANCH=${BRANCH:-master}
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.25 AS builder
FROM golang:1.24 AS builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.25 AS builder
FROM golang:1.24 AS builder
RUN apt-get update
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
-1
View File
@@ -7,7 +7,6 @@
[master.maintenance]
# periodically run these scripts are the same as running them from 'weed shell'
# Scripts are skipped while an admin server is connected.
scripts = """
lock
ec.encode -fullPercent=95 -quietFor=1h
+35 -35
View File
@@ -1,6 +1,6 @@
module github.com/seaweedfs/seaweedfs
go 1.25.0
go 1.24.9
require (
cloud.google.com/go v0.123.0 // indirect
@@ -26,7 +26,7 @@ require (
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-redsync/redsync/v4 v4.16.0
github.com/go-redsync/redsync/v4 v4.15.0
github.com/go-sql-driver/mysql v1.9.3
github.com/go-zookeeper/zk v1.0.3 // indirect
github.com/golang/protobuf v1.5.4
@@ -63,7 +63,7 @@ require (
github.com/prometheus/client_golang v1.23.2
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.67.2 // indirect
github.com/prometheus/procfs v0.20.1
github.com/prometheus/procfs v0.19.2
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/seaweedfs/goexif v1.0.3
@@ -87,22 +87,22 @@ require (
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.etcd.io/etcd/client/v3 v3.6.7
go.mongodb.org/mongo-driver v1.17.9
go.mongodb.org/mongo-driver v1.17.6
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.45.0
gocloud.dev/pubsub/natspubsub v0.45.0
gocloud.dev v0.44.0
gocloud.dev/pubsub/natspubsub v0.44.0
gocloud.dev/pubsub/rabbitpubsub v0.44.0
golang.org/x/crypto v0.48.0
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
golang.org/x/image v0.36.0
golang.org/x/net v0.49.0
golang.org/x/oauth2 v0.35.0
golang.org/x/sys v0.42.0
golang.org/x/oauth2 v0.34.0
golang.org/x/sys v0.41.0
golang.org/x/text v0.34.0 // indirect
golang.org/x/tools v0.41.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/api v0.258.0
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 // indirect
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
google.golang.org/grpc v1.78.0
google.golang.org/protobuf v1.36.11
gopkg.in/inf.v0 v0.9.1 // indirect
@@ -124,13 +124,13 @@ require (
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
github.com/arangodb/go-driver v1.6.9
github.com/armon/go-metrics v0.4.1
github.com/aws/aws-sdk-go-v2 v1.41.3
github.com/aws/aws-sdk-go-v2 v1.41.1
github.com/aws/aws-sdk-go-v2/config v1.32.7
github.com/aws/aws-sdk-go-v2/credentials v1.19.7
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
github.com/cognusion/imaging v1.0.2
github.com/fluent/fluent-logger-golang v1.10.1
github.com/getsentry/sentry-go v0.43.0
github.com/getsentry/sentry-go v0.42.0
github.com/go-ldap/ldap/v3 v3.4.12
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -138,7 +138,8 @@ require (
github.com/hashicorp/raft-boltdb/v2 v2.3.1
github.com/hashicorp/vault/api v1.22.0
github.com/jhump/protoreflect v1.18.0
github.com/linkedin/goavro/v2 v2.15.0
github.com/lib/pq v1.11.1
github.com/linkedin/goavro/v2 v2.14.1
github.com/mattn/go-sqlite3 v1.14.34
github.com/minio/crc64nvme v1.1.1
github.com/orcaman/concurrent-map/v2 v2.0.1
@@ -150,7 +151,7 @@ require (
github.com/redis/go-redis/v9 v9.18.0
github.com/schollz/progressbar/v3 v3.19.0
github.com/seaweedfs/go-fuse/v2 v2.9.1
github.com/shirou/gopsutil/v4 v4.26.2
github.com/shirou/gopsutil/v4 v4.26.1
github.com/tarantool/go-tarantool/v2 v2.4.1
github.com/testcontainers/testcontainers-go v0.39.0
github.com/tikv/client-go/v2 v2.0.7
@@ -171,7 +172,7 @@ require (
atomicgo.dev/keyboard v0.2.9 // indirect
atomicgo.dev/schedule v0.1.0 // indirect
cloud.google.com/go/longrunning v0.7.0 // indirect
cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
dario.cat/mergo v1.0.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
@@ -236,7 +237,6 @@ require (
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.11.1 // indirect
github.com/lithammer/fuzzysearch v1.1.8 // indirect
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
github.com/magiconair/properties v1.8.10 // indirect
@@ -279,10 +279,10 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
go.uber.org/mock v0.5.2 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
@@ -292,12 +292,12 @@ require (
)
require (
cel.dev/expr v0.25.1 // indirect
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go/auth v0.17.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.5.3 // indirect
cloud.google.com/go/monitoring v1.24.3 // indirect
cloud.google.com/go/monitoring v1.24.2 // indirect
filippo.io/edwards25519 v1.1.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
@@ -327,7 +327,7 @@ require (
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -336,12 +336,12 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6
github.com/aws/smithy-go v1.24.2
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
github.com/aws/smithy-go v1.24.0
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
@@ -352,7 +352,7 @@ require (
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e // indirect
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
github.com/creasty/defaults v1.8.0 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect
@@ -360,11 +360,11 @@ require (
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
github.com/ebitengine/purego v0.10.0 // indirect
github.com/ebitengine/purego v0.9.1 // indirect
github.com/elastic/gosigar v0.14.3 // indirect
github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -428,8 +428,8 @@ require (
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/nats.go v1.48.0 // indirect
github.com/nats-io/nkeys v0.4.12 // indirect
github.com/nats-io/nats.go v1.43.0 // indirect
github.com/nats-io/nkeys v0.4.11 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/ncw/swift/v2 v2.0.5 // indirect
@@ -493,11 +493,11 @@ require (
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
go.opentelemetry.io/otel v1.40.0 // indirect
go.opentelemetry.io/otel/metric v1.40.0 // indirect
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
go.opentelemetry.io/otel/trace v1.40.0 // indirect
go.opentelemetry.io/otel v1.38.0 // indirect
go.opentelemetry.io/otel/metric v1.38.0 // indirect
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
go.opentelemetry.io/otel/trace v1.38.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect
golang.org/x/term v0.40.0 // indirect
+80 -84
View File
@@ -6,8 +6,8 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8=
atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ=
atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs=
atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU=
cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc=
cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA=
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
@@ -338,8 +338,8 @@ cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJP
cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk=
cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4=
cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w=
cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM=
cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U=
cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA=
cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o=
cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM=
@@ -393,8 +393,8 @@ cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhz
cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM=
cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk=
cloud.google.com/go/pubsub/v2 v2.3.0 h1:DgAN907x+sP0nScYfBzneRiIhWoXcpCD8ZAut8WX9vs=
cloud.google.com/go/pubsub/v2 v2.3.0/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
cloud.google.com/go/pubsub/v2 v2.2.1 h1:3brZcshL3fIiD1qOxAE2QW9wxsfjioy014x4yC9XuYI=
cloud.google.com/go/pubsub/v2 v2.2.1/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
@@ -504,8 +504,8 @@ cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV6
cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28=
cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y=
cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA=
cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4=
cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI=
cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs=
cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg=
cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos=
@@ -687,8 +687,6 @@ github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUS
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op h1:Ucf+QxEKMbPogRO5guBNe5cgd9uZgfoJLOYs8WWhtjM=
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4=
@@ -716,8 +714,8 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA=
github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
@@ -726,8 +724,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUT
github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 h1:2fjfz3/G9BRvIKuNZ655GwzpklC2kEH0cowZQGO7uBg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4/go.mod h1:Ymws824lvMypLFPwyyUXM52SXuGgxpu0+DISLfKvB+c=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
@@ -748,18 +746,18 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZi
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
@@ -857,8 +855,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e h1:gt7U1Igw0xbJdyaCM5H2CnlAlPSkzrhsebQB6WQWjLA=
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
@@ -974,8 +972,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJtLmY22n99HaZTz+r2Z51xUPi01m3wg=
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg=
github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
@@ -1002,8 +1000,8 @@ github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJ
github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g=
github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98=
github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
@@ -1053,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
@@ -1126,8 +1124,8 @@ github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOr
github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-redsync/redsync/v4 v4.16.0 h1:bNcOzeHH9d3s6pghU9NJFMPrQa41f5Nx3L4YKr3BdEU=
github.com/go-redsync/redsync/v4 v4.16.0/go.mod h1:V4gagqgyASWBZuwx4xGzu72aZNb/6Mo05byUa3mVmKQ=
github.com/go-redsync/redsync/v4 v4.15.0 h1:KH/XymuxSV7vyKs6z1Cxxj+N+N18JlPxgXeP6x4JY54=
github.com/go-redsync/redsync/v4 v4.15.0/go.mod h1:qNp+lLs3vkfZbtA/aM/OjlZHfEr5YTAYhRktFPKHC7s=
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
@@ -1239,8 +1237,6 @@ github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV
github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI=
github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -1518,8 +1514,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/linkedin/goavro/v2 v2.15.0 h1:pDj1UrjUOO62iXhgBiE7jQkpNIc5/tA5eZsgolMjgVI=
github.com/linkedin/goavro/v2 v2.15.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkFS2iI=
github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
github.com/linxGnu/grocksdb v1.10.7 h1:fCi4qvZWo04VgFwGWmO8HQJgUVounJBy+C2TMVPU/ho=
github.com/linxGnu/grocksdb v1.10.7/go.mod h1:OLQKZwiKwaJiAVCsOzWKvwiLwfZ5Vz8Md5TYR7t7pM8=
github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=
@@ -1567,8 +1563,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -1626,14 +1622,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
github.com/nats-io/nats-server/v2 v2.11.12 h1:jGDXTkcjqQ5fCRstwIxvv1K0RHfftFUoSCT/iIZcqOc=
github.com/nats-io/nats-server/v2 v2.11.12/go.mod h1:5MCp/pqm5SEfsvVZ31ll1088ZTwEUdvRX1Hmh/mTTDg=
github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U=
github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak=
github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU=
github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0=
github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug=
github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
@@ -1769,8 +1765,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI=
github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg=
github.com/pterm/pterm v0.12.30/go.mod h1:MOqLIyMOgmTDz9yorcYbcw+HsgoZo3BQfg2wtl3HEFE=
@@ -1804,10 +1800,10 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
github.com/redis/rueidis v1.0.71 h1:pODtnAR5GAB7j4ekhldZ29HKOxe4Hph0GTDGk1ayEQY=
github.com/redis/rueidis v1.0.71/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ=
github.com/redis/rueidis/rueidiscompat v1.0.71 h1:wNZ//kEjMZgBM0KCk7ncOX8KmAgROU2kDdDNpwheG4w=
github.com/redis/rueidis/rueidiscompat v1.0.71/go.mod h1:esmCLJvaRzZoKlgB82G1bY7Iky5TnO9Rz+NlhbEccFI=
github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
github.com/redis/rueidis/rueidiscompat v1.0.69/go.mod h1:iC4Y8DoN0Uth0Uezg9e2trvNRC7QAgGeuP2OPLb5ccI=
github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM=
github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
@@ -1861,8 +1857,8 @@ github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQ
github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
github.com/shirou/gopsutil/v4 v4.26.1 h1:TOkEyriIXk2HX9d4isZJtbjXbEjf5qyKPAzbzY0JWSo=
github.com/shirou/gopsutil/v4 v4.26.1/go.mod h1:medLI9/UNAb0dOI9Q3/7yWSqKkj00u+1tgY8nvv41pc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
@@ -2113,8 +2109,8 @@ go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/Ah
go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -2134,35 +2130,35 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0 h1:zG8GlgXCJQd5BU98C0hZnBbElszTmUgCNCfYneaDL0A=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0/go.mod h1:hOfBCz8kv/wuq73Mx2H2QnWokh/kHZxkh6SNF2bdKtw=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 h1:ZsXq73BERAiNuuFXYqP4MR5hBrjXfMGSO+Cx7qoOZiM=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0/go.mod h1:hg1zaDMpyZJuUzjFxFsRYBoccE86tM9Uf4IqNMUxvrY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w=
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@@ -2186,10 +2182,10 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY=
gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI=
gocloud.dev/pubsub/natspubsub v0.45.0 h1:kfCupVejeynIQcS1GaIvkY3Nj/acLlM5yPWyxZN8wJ8=
gocloud.dev/pubsub/natspubsub v0.45.0/go.mod h1:WU5SMDWuF7CCr2U8UpbrEONYYFOmvigCAvTpOOpUvYE=
gocloud.dev v0.44.0 h1:iVyMAqFl2r6xUy7M4mfqwlN+21UpJoEtgHEcfiLMUXs=
gocloud.dev v0.44.0/go.mod h1:ZmjROXGdC/eKZLF1N+RujDlFRx3D+4Av2thREKDMVxY=
gocloud.dev/pubsub/natspubsub v0.44.0 h1:1Us76ckkdgtiE1p1rJZ+38b9TQP051bmjAiQlFQzYrM=
gocloud.dev/pubsub/natspubsub v0.44.0/go.mod h1:PvVAGIhL14PWGwWIXX/zAK42ixr2/PKP4Q4yMiAUraQ=
gocloud.dev/pubsub/rabbitpubsub v0.44.0 h1:MpRIO6XJ/JTqrlUWt3CxwDe1LvaiXUVu4sS5cv4f/AM=
gocloud.dev/pubsub/rabbitpubsub v0.44.0/go.mod h1:BB9+qT3r6g4M5+4asiXaEeqw4QAOzsWusO5krYaqkdA=
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
@@ -2380,8 +2376,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -2509,8 +2505,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 h1:O1cMQHRfwNpDfDJerqRoE2oD+AFlyid87D40L/OkkJo=
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8=
@@ -2844,8 +2840,8 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc=
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc=
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 h1:ZdyUkS9po3H7G0tuh955QVyyotWvOD4W0aEapeGeUYk=
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846/go.mod h1:Fk4kyraUvqD7i5H6S43sj2W98fbZa75lpZz/eUyhfO0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.17"
appVersion: "4.13"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.17.0
version: 4.0.413
+1 -17
View File
@@ -23,15 +23,8 @@
#
# Adjust storageClass and sizes to match your cluster's available StorageClasses.
# On OpenShift you can discover them with: oc get storageclass
global:
enableReplication: true
# replication type is XYZ:
# X number of replica in other data centers
# Y number of replica in other racks in the same data center
# Z number of replica in other servers in the same rack
replicationPlacement: "000" # no data replica
master:
replicas: 1
data:
type: "persistentVolumeClaim"
size: "10Gi"
@@ -56,7 +49,6 @@ master:
type: RuntimeDefault
volume:
replicas: 1
dataDirs:
- name: data1
type: "persistentVolumeClaim"
@@ -83,7 +75,6 @@ volume:
type: RuntimeDefault
filer:
replicas: 1
data:
type: "persistentVolumeClaim"
size: "25Gi"
@@ -109,19 +100,12 @@ filer:
# S3 gateway (if enabled)
s3:
enabled: true
replicas: 1
port: 8333
enableAuth: true
podSecurityContext:
enabled: true
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
# controller assign them automatically based on the namespace's SCC.
runAsNonRoot: true
logs:
type: "emptyDir"
containerSecurityContext:
enabled: true
allowPrivilegeEscalation: false
@@ -118,10 +118,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.admin "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.admin.extraEnvironmentVars }}
{{- range $key, $value := .Values.admin.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -130,6 +128,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -243,7 +243,8 @@ spec:
{{- if $httpsPort }}
-s3.port.https={{ $httpsPort }} \
{{- end }}
{{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
-s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
-s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
{{- end }}
{{- if or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }}
-s3.config=/etc/sw/s3/seaweedfs_s3_config \
@@ -345,9 +346,6 @@ spec:
- name: client-cert
mountPath: /usr/local/share/ca-certificates/client/
readOnly: true
{{- if .Values.allInOne.s3.enabled }}
{{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }}
{{- end }}
{{- end }}
{{ tpl .Values.allInOne.extraVolumeMounts . | nindent 12 }}
ports:
@@ -475,9 +473,6 @@ spec:
- name: client-cert
secret:
secretName: {{ include "seaweedfs.fullname" . }}-client-cert
{{- if .Values.allInOne.s3.enabled }}
{{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }}
{{- end }}
{{- end }}
{{ tpl .Values.allInOne.extraVolumes . | nindent 8 }}
{{- if .Values.allInOne.nodeSelector }}
@@ -17,9 +17,6 @@ metadata:
spec:
type: {{ .Values.allInOne.service.type | default "ClusterIP" }}
internalTrafficPolicy: {{ .Values.allInOne.service.internalTrafficPolicy | default "Cluster" }}
{{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) .Values.allInOne.s3.trafficDistribution }}
trafficDistribution: {{ include "seaweedfs.trafficDistribution" (dict "value" .Values.allInOne.s3.trafficDistribution "Capabilities" .Capabilities) }}
{{- end }}
ports:
# Master ports
- name: "swfs-master"
@@ -6,10 +6,6 @@ metadata:
name: {{ .Values.cosi.bucketClassName }}
driverName: {{ .Values.cosi.driverName }}
deletionPolicy: Delete
{{- with .Values.cosi.bucketClassParameters }}
parameters:
{{- toYaml . | nindent 2 }}
{{- end }}
---
kind: BucketAccessClass
apiVersion: objectstorage.k8s.io/v1alpha1
@@ -96,10 +96,8 @@ spec:
- name: WEED_GRPC_CA
value: /usr/local/share/ca-certificates/client/ca.crt
{{- end }}
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.cosi "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.cosi.extraEnvironmentVars }}
{{- range $key, $value := .Values.cosi.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -108,6 +106,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
volumeMounts:
- mountPath: /var/lib/cosi
name: socket
@@ -114,10 +114,8 @@ spec:
optional: true
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.filer "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.filer.extraEnvironmentVars }}
{{- range $key, $value := .Values.filer.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -126,6 +124,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.filer.secretExtraEnvironmentVars }}
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
- name: {{ $key }}
@@ -200,7 +210,8 @@ spec:
{{- if .Values.filer.s3.httpsPort }}
-s3.port.https={{ .Values.filer.s3.httpsPort }} \
{{- end }}
{{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
-s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
-s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
{{- end }}
{{- if .Values.filer.s3.enableAuth }}
-s3.config=/etc/sw/seaweedfs_s3_config \
@@ -253,9 +264,6 @@ spec:
- name: client-cert
readOnly: true
mountPath: /usr/local/share/ca-certificates/client
{{- if .Values.filer.s3.enabled }}
{{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }}
{{- end }}
{{- end }}
{{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }}
ports:
@@ -386,9 +394,6 @@ spec:
- name: client-cert
secret:
secretName: {{ include "seaweedfs.fullname" . }}-client-cert
{{- if .Values.filer.s3.enabled }}
{{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }}
{{- end }}
{{- end }}
{{ tpl .Values.filer.extraVolumes . | indent 8 | trim }}
{{- if .Values.filer.nodeSelector }}
@@ -69,7 +69,9 @@ spec:
priorityClassName: {{ .Values.master.priorityClassName | quote }}
{{- end }}
enableServiceLinks: false
{{- if .Values.global.createClusterRole }}
serviceAccountName: {{ .Values.master.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
{{- end }}
{{- if .Values.master.initContainers }}
initContainers:
{{ tpl .Values.master.initContainers . | nindent 8 | trim }}
@@ -96,10 +98,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.master "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.master.extraEnvironmentVars }}
{{- range $key, $value := .Values.master.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -108,6 +108,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -90,10 +90,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.s3 "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.s3.extraEnvironmentVars }}
{{- range $key, $value := .Values.s3.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -102,6 +100,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -127,7 +137,8 @@ spec:
{{- if .Values.s3.httpsPort }}
-port.https={{ .Values.s3.httpsPort }} \
{{- end }}
{{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
-cert.file=/usr/local/share/ca-certificates/client/tls.crt \
-key.file=/usr/local/share/ca-certificates/client/tls.key \
{{- end }}
{{- if .Values.s3.domainName }}
-domainName={{ .Values.s3.domainName }} \
@@ -175,7 +186,6 @@ spec:
- name: client-cert
readOnly: true
mountPath: /usr/local/share/ca-certificates/client/
{{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }}
{{- end }}
{{ tpl .Values.s3.extraVolumeMounts . | nindent 12 | trim }}
ports:
@@ -267,7 +277,6 @@ spec:
- name: client-cert
secret:
secretName: {{ include "seaweedfs.fullname" . }}-client-cert
{{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }}
{{- end }}
{{ tpl .Values.s3.extraVolumes . | indent 8 | trim }}
{{- if .Values.s3.nodeSelector }}
@@ -16,9 +16,8 @@ metadata:
{{- end }}
spec:
internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }}
{{- $td := .Values.s3.trafficDistribution | default .Values.filer.s3.trafficDistribution }}
{{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) $td }}
trafficDistribution: {{ include "seaweedfs.trafficDistribution" (dict "value" $td "Capabilities" .Capabilities) }}
{{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) (or .Values.s3.trafficDistribution .Values.filer.s3.trafficDistribution) }}
trafficDistribution: {{ include "seaweedfs.trafficDistribution" . }}
{{- end }}
ports:
- name: "swfs-s3"
@@ -90,10 +90,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.sftp "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.sftp.extraEnvironmentVars }}
{{- range $key, $value := .Values.sftp.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -102,6 +100,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -59,18 +59,6 @@ Inject extra environment vars in the format key:value, if populated
{{- end -}}
{{- end -}}
{{- define "seaweedfs.mergeExtraEnvironmentVars" -}}
{{- $global := ((.global | default dict).extraEnvironmentVars | default dict) -}}
{{- $component := ((.component | default dict).extraEnvironmentVars | default dict) -}}
{{- $target := .target -}}
{{- range $key, $value := $global }}
{{- $_ := set $target $key $value }}
{{- end }}
{{- range $key, $value := $component }}
{{- $_ := set $target $key $value }}
{{- end }}
{{- end -}}
{{/* Return the proper filer image */}}
{{- define "filer.image" -}}
{{- if .Values.filer.imageOverride -}}
@@ -338,41 +326,11 @@ Create the name of the service account to use
{{- .Values.global.serviceAccountName | default "seaweedfs" -}}
{{- end -}}
{{/* S3 TLS cert/key arguments, using custom secret if s3.tlsSecret is set */}}
{{- define "seaweedfs.s3.tlsArgs" -}}
{{- $prefix := .prefix -}}
{{- $root := .root -}}
{{- if $root.Values.s3.tlsSecret -}}
-{{ $prefix }}cert.file=/usr/local/share/ca-certificates/s3/tls.crt \
-{{ $prefix }}key.file=/usr/local/share/ca-certificates/s3/tls.key \
{{- else -}}
-{{ $prefix }}cert.file=/usr/local/share/ca-certificates/client/tls.crt \
-{{ $prefix }}key.file=/usr/local/share/ca-certificates/client/tls.key \
{{- end -}}
{{- end -}}
{{/* S3 custom TLS volume mount */}}
{{- define "seaweedfs.s3.tlsVolumeMount" -}}
{{- if .Values.s3.tlsSecret }}
- name: s3-tls-cert
readOnly: true
mountPath: /usr/local/share/ca-certificates/s3/
{{- end }}
{{- end -}}
{{/* S3 custom TLS volume */}}
{{- define "seaweedfs.s3.tlsVolume" -}}
{{- if .Values.s3.tlsSecret }}
- name: s3-tls-cert
secret:
secretName: {{ .Values.s3.tlsSecret }}
{{- end }}
{{- end -}}
{{/* Generate a compatible trafficDistribution value due to "PreferClose" fast deprecation in k8s v1.35.
Accepts a dict with "value" (the trafficDistribution string) and "Capabilities". */}}
{{/* Generate a compatible trafficDistribution value due to "PreferClose" fast deprecation in k8s v1.35 */}}
{{- define "seaweedfs.trafficDistribution" -}}
{{- if .value -}}
{{- and (eq .value "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .value -}}
{{- if .Values.s3.trafficDistribution -}}
{{- and (eq .Values.s3.trafficDistribution "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .Values.s3.trafficDistribution -}}
{{- else if .Values.filer.s3.trafficDistribution -}}
{{- and (eq .Values.filer.s3.trafficDistribution "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .Values.filer.s3.trafficDistribution -}}
{{- end -}}
{{- end -}}
@@ -3,15 +3,6 @@
{{- $s3Enabled := false }}
{{- $enableAuth := false }}
{{- $existingConfigSecret := "" }}
{{- $bucketsFolder := "/buckets" }}
{{- $bucketEnvVars := merge (dict) (.Values.global.extraEnvironmentVars | default dict) }}
{{- if .Values.allInOne.enabled }}
{{- $bucketEnvVars = merge (.Values.allInOne.extraEnvironmentVars | default dict) $bucketEnvVars }}
{{- else }}
{{- $bucketEnvVars = merge (.Values.filer.extraEnvironmentVars | default dict) $bucketEnvVars }}
{{- end }}
{{- $bucketsFolder = default $bucketsFolder (get $bucketEnvVars "WEED_FILER_BUCKETS_FOLDER") }}
{{- $bucketsFolder = trimSuffix "/" $bucketsFolder }}
{{- /* Check allInOne mode first */}}
{{- if .Values.allInOne.enabled }}
@@ -51,7 +42,7 @@ metadata:
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "-5"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
"helm.sh/hook-delete-policy": hook-succeeded
spec:
template:
metadata:
@@ -92,7 +83,6 @@ spec:
- "/bin/sh"
- "-ec"
- |
set -o pipefail
wait_for_service() {
local url=$1
local max_attempts=60 # 5 minutes total (5s * 60)
@@ -118,51 +108,15 @@ spec:
wait_for_service "http://$WEED_CLUSTER_SW_MASTER{{ .Values.master.readinessProbe.httpGet.path }}"
wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}"
{{- end }}
{{- range $createBuckets }}
{{- $bucketName := .name }}
{{- $bucketLock := or .lock .objectLock .withLock }}
bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }
if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then
echo "Bucket '{{ $bucketName }}' already exists, skipping creation."
{{- range $createBuckets }}
if /bin/echo "s3.bucket.list" | /usr/bin/weed shell | awk '{print $1}' | grep -Fxq "{{ .name }}"; then
echo "Bucket '{{ .name }}' already exists, skipping creation."
else
echo "Creating bucket '{{ $bucketName }}'..."
/bin/echo 's3.bucket.create --name {{ $bucketName }}{{- if $bucketLock }} --withLock{{- end }}' | /usr/bin/weed shell
echo "Creating bucket '{{ .name }}'..."
/bin/echo "s3.bucket.create --name {{ .name }}" | /usr/bin/weed shell
fi
{{- end }}
{{- range $createBuckets }}
{{- $bucketLock := or .lock .objectLock .withLock }}
{{- if $bucketLock }}
/bin/echo 's3.bucket.lock -name {{ .name }} -enable' | /usr/bin/weed shell
{{- end }}
{{- end }}
{{- range $createBuckets }}
{{- $bucketVersioning := "" }}
{{- if kindIs "bool" .versioning }}
{{- if .versioning }}
{{- $bucketVersioning = "Enabled" }}
{{- end }}
{{- else if kindIs "string" .versioning }}
{{- $versioningLower := lower .versioning }}
{{- if eq $versioningLower "enabled" "enable" "true" }}
{{- $bucketVersioning = "Enabled" }}
{{- else if eq $versioningLower "suspended" "disable" "false" }}
{{- $bucketVersioning = "Suspended" }}
{{- else if or (eq .versioning "Enabled") (eq .versioning "Suspended") }}
{{- $bucketVersioning = .versioning }}
{{- else }}
{{- fail (printf "Invalid versioning value for bucket %s: %s. Must be 'Enabled' or 'Suspended'" .name .versioning) }}
{{- end }}
{{- end }}
{{- if $bucketVersioning }}
/bin/echo 's3.bucket.versioning -name {{ .name }} -status {{ $bucketVersioning }}' | /usr/bin/weed shell
{{- end }}
{{- end }}
{{- range $createBuckets }}
{{- if .ttl }}
/bin/echo 'fs.configure -locationPrefix={{ $bucketsFolder }}/{{ .name }}/ -ttl={{ .ttl }} -apply' | /usr/bin/weed shell
{{- end }}
{{- end }}
{{- range $createBuckets }}
{{- if .anonymousRead }}
/bin/echo \
"s3.configure --user anonymous \
@@ -69,7 +69,9 @@ spec:
priorityClassName: {{ $volume.priorityClassName | quote }}
{{- end }}
enableServiceLinks: false
{{- if $.Values.global.createClusterRole }}
serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
{{- end }}
{{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
{{- if $initContainers_exists }}
initContainers:
@@ -116,10 +118,8 @@ spec:
fieldPath: status.hostIP
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" $ }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" $.Values.global "component" $volume "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if $volume.extraEnvironmentVars }}
{{- range $key, $value := $volume.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -128,6 +128,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if $.Values.global.extraEnvironmentVars }}
{{- range $key, $value := $.Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -93,10 +93,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.worker "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.worker.extraEnvironmentVars }}
{{- range $key, $value := .Values.worker.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -105,6 +103,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
-28
View File
@@ -898,15 +898,9 @@ filer:
auditLogConfig: {}
# You may specify buckets to be created during the install or upgrade process.
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
# objectLock enables S3 Object Lock (irreversible, forces versioning)
# versioning: Enabled or Suspended (or true to enable)
# createBuckets:
# - name: bucket-a
# anonymousRead: true
# ttl: 7d
# objectLock: true
# versioning: Enabled
# - name: bucket-b
# anonymousRead: false
@@ -919,13 +913,6 @@ s3:
port: 8333
# add additional https port
httpsPort: 0
# Use a custom TLS certificate secret for the S3 HTTPS endpoint.
# When set, this Kubernetes Secret (must contain tls.crt and tls.key) is used
# instead of the internal self-signed client certificate generated by cert-manager.
# This allows using a publicly trusted certificate (e.g., from Let's Encrypt)
# so that S3 clients don't need to trust the internal CA.
# Requires global.enableSecurity to be true.
tlsSecret: null
metricsPort: 9327
# Iceberg catalog REST port (Apache Iceberg REST Catalog API)
# Set to a port number to enable, or 0/null to disable
@@ -949,15 +936,9 @@ s3:
auditLogConfig: {}
# You may specify buckets to be created during the install or upgrade process.
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
# objectLock enables S3 Object Lock (irreversible, forces versioning)
# versioning: Enabled or Suspended (or true to enable)
# createBuckets:
# - name: bucket-a
# anonymousRead: true
# ttl: 7d
# objectLock: true
# versioning: Enabled
# - name: bucket-b
# anonymousRead: false
@@ -1460,18 +1441,11 @@ allInOne:
# The s3-secret.yaml template only reads from .Values.s3.credentials.
# See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig)
trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35
# You may specify buckets to be created during the install process.
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
# objectLock enables S3 Object Lock (irreversible, forces versioning)
# versioning: Enabled or Suspended (or true to enable)
# createBuckets:
# - name: bucket-a
# anonymousRead: true
# ttl: 7d
# objectLock: true
# versioning: Enabled
# - name: bucket-b
# anonymousRead: false
@@ -1637,8 +1611,6 @@ cosi:
image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.1.2"
driverName: "seaweedfs.objectstorage.k8s.io"
bucketClassName: "seaweedfs"
# Optional parameters to pass to the default BucketClass (e.g., diskType for tiered storage)
bucketClassParameters: {}
endpoint: ""
region: ""
@@ -100,12 +100,10 @@ message ListEntriesRequest {
string startFromFileName = 3;
bool inclusiveStartFrom = 4;
uint32 limit = 5;
int64 snapshot_ts_ns = 6;
}
message ListEntriesResponse {
Entry entry = 1;
int64 snapshot_ts_ns = 2;
}
message RemoteEntry {
@@ -205,7 +203,6 @@ message CreateEntryRequest {
message CreateEntryResponse {
string error = 1;
SubscribeMetadataResponse metadata_event = 2;
}
message UpdateEntryRequest {
@@ -215,7 +212,6 @@ message UpdateEntryRequest {
repeated int32 signatures = 4;
}
message UpdateEntryResponse {
SubscribeMetadataResponse metadata_event = 1;
}
message AppendToEntryRequest {
@@ -240,7 +236,6 @@ message DeleteEntryRequest {
message DeleteEntryResponse {
string error = 1;
SubscribeMetadataResponse metadata_event = 2;
}
message AtomicRenameEntryRequest {
@@ -474,7 +469,6 @@ message CacheRemoteObjectToLocalClusterRequest {
}
message CacheRemoteObjectToLocalClusterResponse {
Entry entry = 1;
SubscribeMetadataResponse metadata_event = 2;
}
/////////////////////////
+1 -1
View File
@@ -1,5 +1,5 @@
# Multi-stage build for Go Sidecar
FROM golang:1.25-alpine AS builder
FROM golang:1.24-alpine AS builder
# Install build dependencies
RUN apk add --no-cache git ca-certificates tzdata
@@ -1,5 +1,5 @@
# Multi-stage build for Test Client
FROM golang:1.25-alpine AS builder
FROM golang:1.23-alpine AS builder
# Install build dependencies
RUN apk add --no-cache git ca-certificates tzdata
@@ -153,38 +153,6 @@ func waitForUrl(t *testing.T, url string, retries int) {
t.Fatalf("Timeout waiting for %s", url)
}
func fetchJSON(url string, out interface{}) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
}
return json.NewDecoder(resp.Body).Decode(out)
}
func mapField(obj map[string]interface{}, key string) (interface{}, bool) {
if obj == nil {
return nil, false
}
if value, ok := obj[key]; ok {
return value, true
}
return nil, false
}
func mapFieldAny(obj map[string]interface{}, keys ...string) (interface{}, bool) {
for _, key := range keys {
if value, ok := mapField(obj, key); ok {
return value, true
}
}
return nil, false
}
func TestEcEndToEnd(t *testing.T) {
defer cleanup()
ensureEnvironment(t)
@@ -307,7 +275,6 @@ func TestEcEndToEnd(t *testing.T) {
startTime := time.Now()
ecVerified := false
var lastBody []byte
debugTick := 0
for time.Since(startTime) < 300*time.Second {
// 3.1 Check Master Topology
@@ -333,104 +300,25 @@ func TestEcEndToEnd(t *testing.T) {
}
}
// 3.2 Debug: Check workers, jobs, and scheduler status
debugTick++
var workers []map[string]interface{}
// 3.2 Debug: Check workers and jobs
wResp, wErr := http.Get(AdminUrl + "/api/plugin/workers")
workerCount := 0
ecDetectorCount := 0
ecExecutorCount := 0
if err := fetchJSON(AdminUrl+"/api/plugin/workers", &workers); err == nil {
if wErr == nil {
var workers []interface{}
json.NewDecoder(wResp.Body).Decode(&workers)
wResp.Body.Close()
workerCount = len(workers)
for _, worker := range workers {
capsValue, ok := mapFieldAny(worker, "capabilities", "Capabilities")
if !ok {
continue
}
caps, ok := capsValue.(map[string]interface{})
if !ok {
continue
}
if capValue, ok := caps["erasure_coding"].(map[string]interface{}); ok {
if capValue["can_detect"] == true {
ecDetectorCount++
}
if capValue["can_execute"] == true {
ecExecutorCount++
}
}
}
}
var tasks []map[string]interface{}
tResp, tErr := http.Get(AdminUrl + "/api/plugin/jobs?limit=1000")
taskCount := 0
ecTaskCount := 0
ecTaskStates := map[string]int{}
if err := fetchJSON(AdminUrl+"/api/plugin/jobs?limit=1000", &tasks); err == nil {
if tErr == nil {
var tasks []interface{}
json.NewDecoder(tResp.Body).Decode(&tasks)
tResp.Body.Close()
taskCount = len(tasks)
for _, task := range tasks {
jobType, _ := task["job_type"].(string)
state, _ := task["state"].(string)
if jobType == "erasure_coding" {
ecTaskCount++
ecTaskStates[state]++
}
}
}
t.Logf("Waiting for EC... (Workers: %d det=%d exec=%d, Tasks: %d ec=%d, EC States: %+v)",
workerCount, ecDetectorCount, ecExecutorCount, taskCount, ecTaskCount, ecTaskStates)
if debugTick%3 == 0 {
var pluginStatus map[string]interface{}
if err := fetchJSON(AdminUrl+"/api/plugin/status", &pluginStatus); err == nil {
t.Logf("Plugin status: enabled=%v worker_count=%v worker_grpc_port=%v configured=%v",
pluginStatus["enabled"], pluginStatus["worker_count"], pluginStatus["worker_grpc_port"], pluginStatus["configured"])
}
var schedulerStatus map[string]interface{}
if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-status", &schedulerStatus); err == nil {
if schedValue, ok := schedulerStatus["scheduler"].(map[string]interface{}); ok {
t.Logf("Scheduler status: current_job_type=%v phase=%v last_iteration_had_jobs=%v idle_sleep_seconds=%v last_iteration_done_at=%v next_detection_at=%v",
schedValue["current_job_type"], schedValue["current_phase"],
schedValue["last_iteration_had_jobs"], schedValue["idle_sleep_seconds"], schedValue["last_iteration_done_at"], schedValue["next_detection_at"])
} else {
t.Logf("Scheduler status: %v", schedulerStatus)
}
}
var schedulerStates []map[string]interface{}
if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-states", &schedulerStates); err == nil {
for _, state := range schedulerStates {
if state["job_type"] == "erasure_coding" {
t.Logf("EC scheduler state: enabled=%v detection_in_flight=%v detector_available=%v executor_workers=%v next_detection_at=%v last_run_status=%v last_run_started_at=%v last_run_completed_at=%v",
state["enabled"], state["detection_in_flight"], state["detector_available"],
state["executor_worker_count"], state["next_detection_at"], state["last_run_status"],
state["last_run_started_at"], state["last_run_completed_at"])
break
}
}
}
var jobTypes []map[string]interface{}
if err := fetchJSON(AdminUrl+"/api/plugin/job-types", &jobTypes); err == nil {
var names []string
for _, jobType := range jobTypes {
if name, ok := jobType["job_type"].(string); ok && name != "" {
names = append(names, name)
}
}
t.Logf("Plugin job types: %v", names)
}
var activities []map[string]interface{}
if err := fetchJSON(AdminUrl+"/api/plugin/activities?job_type=erasure_coding&limit=5", &activities); err == nil {
for i := len(activities) - 1; i >= 0; i-- {
act := activities[i]
t.Logf("EC activity: stage=%v message=%v occurred_at=%v", act["stage"], act["message"], act["occurred_at"])
}
}
}
t.Logf("Waiting for EC... (Workers: %d, Active Tasks: %d)", workerCount, taskCount)
time.Sleep(10 * time.Second)
}
+1 -1
View File
@@ -1,6 +1,6 @@
# Simplified single-stage build for SeaweedFS with FoundationDB support
# Force x86_64 platform to use AMD64 FoundationDB packages
FROM --platform=linux/amd64 golang:1.25-bookworm
FROM --platform=linux/amd64 golang:1.24-bookworm
ARG FOUNDATIONDB_VERSION=7.4.5
ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}
+1 -1
View File
@@ -1,5 +1,5 @@
# Multi-stage Dockerfile to build SeaweedFS with FoundationDB support for ARM64
FROM --platform=linux/arm64 golang:1.25-bookworm AS builder
FROM --platform=linux/arm64 golang:1.24-bookworm AS builder
ARG FOUNDATIONDB_VERSION=7.4.5
ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}
+1 -1
View File
@@ -1,5 +1,5 @@
# Test environment with Go and FoundationDB support
FROM golang:1.25-bookworm
FROM golang:1.24-bookworm
# Install system dependencies
RUN apt-get update && apt-get install -y \
+2 -2
View File
@@ -21,7 +21,7 @@
<scala.binary.version>2.12</scala.binary.version>
<junit.version>4.13.2</junit.version>
<seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
<jackson.version>2.18.6</jackson.version> <!-- Upgraded from 2.15.3 -->
<jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
<netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
<parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
<parquet.format.version>2.12.0</parquet.format.version>
@@ -123,7 +123,7 @@
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.9.5</version>
<version>3.9.4</version>
</dependency>
<!-- Apache Commons - Fix CVEs -->
+1 -1
View File
@@ -1,5 +1,5 @@
# Dockerfile for Kafka Gateway Integration Testing
FROM golang:1.25-alpine AS builder
FROM golang:1.24-alpine AS builder
# Install build dependencies
RUN apk add --no-cache git make gcc musl-dev sqlite-dev
+1 -1
View File
@@ -1,5 +1,5 @@
# Dockerfile for building SeaweedFS components from the current workspace
FROM golang:1.25-alpine AS builder
FROM golang:1.24-alpine AS builder
RUN apk add --no-cache git make gcc musl-dev sqlite-dev
+1 -1
View File
@@ -1,5 +1,5 @@
# Dockerfile for Kafka Integration Test Setup
FROM golang:1.25-alpine AS builder
FROM golang:1.24-alpine AS builder
# Install build dependencies
RUN apk add --no-cache git make gcc musl-dev
@@ -2,7 +2,7 @@
# Multi-stage build for cross-platform support
# Stage 1: Builder
FROM golang:1.25-alpine AS builder
FROM golang:1.24-alpine AS builder
WORKDIR /app
@@ -0,0 +1,78 @@
package ec_repair_test
import (
"context"
"testing"
"time"
pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
func TestEcRepairDetectionFindsCandidates(t *testing.T) {
const (
volumeID = uint32(100)
collection = "ec-test"
diskType = "hdd"
)
node1 := nodeSpec{
id: "node1",
address: "127.0.0.1:11001",
diskType: diskType,
diskID: 0,
ecShards: []*master_pb.VolumeEcShardInformationMessage{
buildEcShardInfo(volumeID, collection, diskType, 0, map[uint32]int64{
0: 100,
1: 100,
2: 100,
3: 100,
4: 100,
}),
},
}
node2 := nodeSpec{
id: "node2",
address: "127.0.0.1:11002",
diskType: diskType,
diskID: 0,
ecShards: []*master_pb.VolumeEcShardInformationMessage{
buildEcShardInfo(volumeID, collection, diskType, 0, map[uint32]int64{
0: 50,
5: 100,
6: 100,
7: 100,
8: 100,
9: 100,
}),
},
}
topo := buildTopology([]nodeSpec{node1, node2})
response := &master_pb.VolumeListResponse{TopologyInfo: topo}
master := pluginworkers.NewMasterServer(t, response)
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
handler := pluginworker.NewEcRepairHandler(dialOption)
harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
WorkerOptions: pluginworker.WorkerOptions{GrpcDialOption: dialOption},
Handlers: []pluginworker.JobHandler{handler},
})
harness.WaitForJobType("ec_repair")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
proposals, err := harness.Plugin().RunDetection(ctx, "ec_repair", &plugin_pb.ClusterContext{
MasterGrpcAddresses: []string{master.Address()},
}, 10)
require.NoError(t, err)
require.NotEmpty(t, proposals)
require.Equal(t, "ec_repair", proposals[0].JobType)
}
@@ -0,0 +1,131 @@
package ec_repair_test
import (
"context"
"fmt"
"testing"
"time"
pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
func TestEcRepairExecutionRepairsShards(t *testing.T) {
const (
volumeID = uint32(200)
collection = "ec-repair"
diskType = "hdd"
)
volumeServers := []*pluginworkers.VolumeServer{
pluginworkers.NewVolumeServer(t, ""),
pluginworkers.NewVolumeServer(t, ""),
pluginworkers.NewVolumeServer(t, ""),
pluginworkers.NewVolumeServer(t, ""),
}
nodes := []nodeSpec{
{
id: "node1",
address: volumeServers[0].Address(),
diskType: diskType,
diskID: 0,
ecShards: []*master_pb.VolumeEcShardInformationMessage{
buildEcShardInfo(volumeID, collection, diskType, 0, map[uint32]int64{
0: 100,
1: 100,
2: 100,
3: 100,
4: 100,
}),
},
},
{
id: "node2",
address: volumeServers[1].Address(),
diskType: diskType,
diskID: 0,
ecShards: []*master_pb.VolumeEcShardInformationMessage{
buildEcShardInfo(volumeID, collection, diskType, 0, map[uint32]int64{
0: 50,
5: 100,
6: 100,
7: 100,
8: 100,
9: 100,
}),
},
},
{
id: "node3",
address: volumeServers[2].Address(),
diskType: diskType,
diskID: 0,
},
{
id: "node4",
address: volumeServers[3].Address(),
diskType: diskType,
diskID: 0,
},
}
topo := buildTopology(nodes)
response := &master_pb.VolumeListResponse{TopologyInfo: topo}
master := pluginworkers.NewMasterServer(t, response)
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
handler := pluginworker.NewEcRepairHandler(dialOption)
harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
WorkerOptions: pluginworker.WorkerOptions{GrpcDialOption: dialOption},
Handlers: []pluginworker.JobHandler{handler},
})
harness.WaitForJobType("ec_repair")
job := &plugin_pb.JobSpec{
JobId: fmt.Sprintf("ec-repair-%d", volumeID),
JobType: "ec_repair",
Parameters: map[string]*plugin_pb.ConfigValue{
"volume_id": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
},
"collection": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: collection},
},
"disk_type": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: diskType},
},
},
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result, err := harness.Plugin().ExecuteJob(ctx, job, &plugin_pb.ClusterContext{
MasterGrpcAddresses: []string{master.Address()},
}, 1)
require.NoError(t, err)
require.NotNil(t, result)
require.True(t, result.Success)
rebuildCalls := 0
copyCalls := 0
deleteCalls := 0
unmountCalls := 0
for _, vs := range volumeServers {
rebuildCalls += len(vs.RebuildRequests())
copyCalls += len(vs.CopyRequests())
deleteCalls += len(vs.EcDeleteRequests())
unmountCalls += len(vs.UnmountRequests())
}
require.Greater(t, rebuildCalls, 0)
require.Greater(t, copyCalls, 0)
require.Greater(t, deleteCalls, 0)
require.Greater(t, unmountCalls, 0)
}
@@ -0,0 +1,69 @@
package ec_repair_test
import (
"sort"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
)
type nodeSpec struct {
id string
address string
diskType string
diskID uint32
ecShards []*master_pb.VolumeEcShardInformationMessage
}
func buildTopology(nodes []nodeSpec) *master_pb.TopologyInfo {
dataNodes := make([]*master_pb.DataNodeInfo, 0, len(nodes))
for _, node := range nodes {
diskInfo := &master_pb.DiskInfo{
DiskId: node.diskID,
MaxVolumeCount: 100,
VolumeCount: 10,
EcShardInfos: node.ecShards,
}
dataNodes = append(dataNodes, &master_pb.DataNodeInfo{
Id: node.id,
Address: node.address,
DiskInfos: map[string]*master_pb.DiskInfo{node.diskType: diskInfo},
})
}
return &master_pb.TopologyInfo{
DataCenterInfos: []*master_pb.DataCenterInfo{
{
Id: "dc1",
RackInfos: []*master_pb.RackInfo{
{
Id: "rack1",
DataNodeInfos: dataNodes,
},
},
},
},
}
}
func buildEcShardInfo(volumeID uint32, collection, diskType string, diskID uint32, shardSizes map[uint32]int64) *master_pb.VolumeEcShardInformationMessage {
shardIDs := make([]int, 0, len(shardSizes))
for shardID := range shardSizes {
shardIDs = append(shardIDs, int(shardID))
}
sort.Ints(shardIDs)
var bits uint32
sizes := make([]int64, 0, len(shardIDs))
for _, shardID := range shardIDs {
bits |= (1 << shardID)
sizes = append(sizes, shardSizes[uint32(shardID)])
}
return &master_pb.VolumeEcShardInformationMessage{
Id: volumeID,
Collection: collection,
EcIndexBits: bits,
DiskType: diskType,
DiskId: diskID,
ShardSizes: sizes,
}
}
+85 -48
View File
@@ -27,21 +27,23 @@ type VolumeServer struct {
address string
baseDir string
mu sync.Mutex
receivedFiles map[string]uint64
mountRequests []*volume_server_pb.VolumeEcShardsMountRequest
deleteRequests []*volume_server_pb.VolumeDeleteRequest
markReadonlyCalls int
markWritableCalls int
readFileStatusCalls int
vacuumGarbageRatio float64
vacuumCheckCalls int
vacuumCompactCalls int
vacuumCommitCalls int
vacuumCleanupCalls int
volumeCopyCalls int
volumeMountCalls int
tailReceiverCalls int
mu sync.Mutex
receivedFiles map[string]uint64
mountRequests []*volume_server_pb.VolumeEcShardsMountRequest
unmountRequests []*volume_server_pb.VolumeEcShardsUnmountRequest
copyRequests []*volume_server_pb.VolumeEcShardsCopyRequest
rebuildRequests []*volume_server_pb.VolumeEcShardsRebuildRequest
ecDeleteRequests []*volume_server_pb.VolumeEcShardsDeleteRequest
deleteRequests []*volume_server_pb.VolumeDeleteRequest
markReadonlyCalls int
vacuumGarbageRatio float64
vacuumCheckCalls int
vacuumCompactCalls int
vacuumCommitCalls int
vacuumCleanupCalls int
volumeCopyCalls int
volumeMountCalls int
tailReceiverCalls int
}
// NewVolumeServer starts a test volume server using the provided base directory.
@@ -136,6 +138,46 @@ func (v *VolumeServer) MountRequests() []*volume_server_pb.VolumeEcShardsMountRe
return out
}
// UnmountRequests returns recorded unmount requests.
func (v *VolumeServer) UnmountRequests() []*volume_server_pb.VolumeEcShardsUnmountRequest {
v.mu.Lock()
defer v.mu.Unlock()
out := make([]*volume_server_pb.VolumeEcShardsUnmountRequest, len(v.unmountRequests))
copy(out, v.unmountRequests)
return out
}
// CopyRequests returns recorded EC shard copy requests.
func (v *VolumeServer) CopyRequests() []*volume_server_pb.VolumeEcShardsCopyRequest {
v.mu.Lock()
defer v.mu.Unlock()
out := make([]*volume_server_pb.VolumeEcShardsCopyRequest, len(v.copyRequests))
copy(out, v.copyRequests)
return out
}
// RebuildRequests returns recorded EC shard rebuild requests.
func (v *VolumeServer) RebuildRequests() []*volume_server_pb.VolumeEcShardsRebuildRequest {
v.mu.Lock()
defer v.mu.Unlock()
out := make([]*volume_server_pb.VolumeEcShardsRebuildRequest, len(v.rebuildRequests))
copy(out, v.rebuildRequests)
return out
}
// EcDeleteRequests returns recorded EC shard delete requests.
func (v *VolumeServer) EcDeleteRequests() []*volume_server_pb.VolumeEcShardsDeleteRequest {
v.mu.Lock()
defer v.mu.Unlock()
out := make([]*volume_server_pb.VolumeEcShardsDeleteRequest, len(v.ecDeleteRequests))
copy(out, v.ecDeleteRequests)
return out
}
// DeleteRequests returns recorded delete requests.
func (v *VolumeServer) DeleteRequests() []*volume_server_pb.VolumeDeleteRequest {
v.mu.Lock()
@@ -153,20 +195,6 @@ func (v *VolumeServer) MarkReadonlyCount() int {
return v.markReadonlyCalls
}
// MarkWritableCount returns the number of writable calls.
func (v *VolumeServer) MarkWritableCount() int {
v.mu.Lock()
defer v.mu.Unlock()
return v.markWritableCalls
}
// ReadFileStatusCount returns the number of ReadVolumeFileStatus calls.
func (v *VolumeServer) ReadFileStatusCount() int {
v.mu.Lock()
defer v.mu.Unlock()
return v.readFileStatusCalls
}
// Shutdown stops the volume server.
func (v *VolumeServer) Shutdown() {
if v.server != nil {
@@ -276,6 +304,34 @@ func (v *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_serv
return &volume_server_pb.VolumeEcShardsMountResponse{}, nil
}
func (v *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) {
v.mu.Lock()
v.unmountRequests = append(v.unmountRequests, req)
v.mu.Unlock()
return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil
}
func (v *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_server_pb.VolumeEcShardsCopyRequest) (*volume_server_pb.VolumeEcShardsCopyResponse, error) {
v.mu.Lock()
v.copyRequests = append(v.copyRequests, req)
v.mu.Unlock()
return &volume_server_pb.VolumeEcShardsCopyResponse{}, nil
}
func (v *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_server_pb.VolumeEcShardsRebuildRequest) (*volume_server_pb.VolumeEcShardsRebuildResponse, error) {
v.mu.Lock()
v.rebuildRequests = append(v.rebuildRequests, req)
v.mu.Unlock()
return &volume_server_pb.VolumeEcShardsRebuildResponse{}, nil
}
func (v *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
v.mu.Lock()
v.ecDeleteRequests = append(v.ecDeleteRequests, req)
v.mu.Unlock()
return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil
}
func (v *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
v.mu.Lock()
v.deleteRequests = append(v.deleteRequests, req)
@@ -296,25 +352,6 @@ func (v *VolumeServer) VolumeMarkReadonly(ctx context.Context, req *volume_serve
return &volume_server_pb.VolumeMarkReadonlyResponse{}, nil
}
func (v *VolumeServer) VolumeMarkWritable(ctx context.Context, req *volume_server_pb.VolumeMarkWritableRequest) (*volume_server_pb.VolumeMarkWritableResponse, error) {
v.mu.Lock()
v.markWritableCalls++
v.mu.Unlock()
return &volume_server_pb.VolumeMarkWritableResponse{}, nil
}
func (v *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest) (*volume_server_pb.ReadVolumeFileStatusResponse, error) {
v.mu.Lock()
v.readFileStatusCalls++
v.mu.Unlock()
return &volume_server_pb.ReadVolumeFileStatusResponse{
VolumeId: req.VolumeId,
DatFileSize: 1024,
IdxFileSize: 16,
FileCount: 1,
}, nil
}
func (v *VolumeServer) VacuumVolumeCheck(ctx context.Context, req *volume_server_pb.VacuumVolumeCheckRequest) (*volume_server_pb.VacuumVolumeCheckResponse, error) {
v.mu.Lock()
v.vacuumCheckCalls++
@@ -37,9 +37,7 @@ func TestVolumeBalanceDetectionIntegration(t *testing.T) {
MasterGrpcAddresses: []string{master.Address()},
}, 10)
require.NoError(t, err)
// With default batch_size=20 and 10 overloaded volumes vs 1 underloaded,
// all moves are grouped into a single batch proposal.
require.Len(t, proposals, 1, "expected exactly one batch proposal")
require.Len(t, proposals, 1)
proposal := proposals[0]
require.Equal(t, "volume_balance", proposal.JobType)
@@ -48,15 +46,8 @@ func TestVolumeBalanceDetectionIntegration(t *testing.T) {
params := &worker_pb.TaskParams{}
require.NoError(t, proto.Unmarshal(paramsValue.GetBytesValue(), params))
bp := params.GetBalanceParams()
require.NotNil(t, bp, "expected BalanceParams in batch proposal")
require.Greater(t, len(bp.Moves), 1, "batch proposal should contain multiple moves")
for _, move := range bp.Moves {
require.NotZero(t, move.VolumeId)
require.NotEmpty(t, move.SourceNode)
require.NotEmpty(t, move.TargetNode)
}
require.NotEmpty(t, params.Sources)
require.NotEmpty(t, params.Targets)
}
func buildBalanceVolumeListResponse(t *testing.T) *master_pb.VolumeListResponse {
@@ -8,12 +8,10 @@ import (
pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/protobuf/proto"
)
func TestVolumeBalanceExecutionIntegration(t *testing.T) {
@@ -62,92 +60,8 @@ func TestVolumeBalanceExecutionIntegration(t *testing.T) {
require.GreaterOrEqual(t, source.MarkReadonlyCount(), 1)
require.GreaterOrEqual(t, len(source.DeleteRequests()), 1)
copyCalls, _, tailCalls := target.BalanceStats()
copyCalls, mountCalls, tailCalls := target.BalanceStats()
require.GreaterOrEqual(t, copyCalls, 1)
require.GreaterOrEqual(t, mountCalls, 1)
require.GreaterOrEqual(t, tailCalls, 1)
}
func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
handler := pluginworker.NewVolumeBalanceHandler(dialOption)
harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
WorkerOptions: pluginworker.WorkerOptions{
GrpcDialOption: dialOption,
},
Handlers: []pluginworker.JobHandler{handler},
})
harness.WaitForJobType("volume_balance")
// Create one source and one target fake volume server.
source := pluginworkers.NewVolumeServer(t, "")
target := pluginworkers.NewVolumeServer(t, "")
// Build a batch job with 3 volume moves from source → target.
volumeIDs := []uint32{401, 402, 403}
moves := make([]*worker_pb.BalanceMoveSpec, len(volumeIDs))
for i, vid := range volumeIDs {
moves[i] = &worker_pb.BalanceMoveSpec{
VolumeId: vid,
SourceNode: source.Address(),
TargetNode: target.Address(),
Collection: "batch-test",
}
}
params := &worker_pb.TaskParams{
TaskId: "batch-balance-test",
TaskParams: &worker_pb.TaskParams_BalanceParams{
BalanceParams: &worker_pb.BalanceTaskParams{
MaxConcurrentMoves: 2,
Moves: moves,
},
},
}
paramBytes, err := proto.Marshal(params)
require.NoError(t, err)
job := &plugin_pb.JobSpec{
JobId: "batch-balance-test",
JobType: "volume_balance",
Parameters: map[string]*plugin_pb.ConfigValue{
"task_params_pb": {
Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramBytes},
},
},
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
result, err := harness.Plugin().ExecuteJob(ctx, job, nil, 1)
require.NoError(t, err)
require.NotNil(t, result)
require.True(t, result.Success, "batch balance job should succeed; result: %+v", result)
// Each of the 3 moves should have marked the source readonly and deleted.
require.Equal(t, len(volumeIDs), source.MarkReadonlyCount(),
"each move should mark source volume readonly")
require.Equal(t, len(volumeIDs), len(source.DeleteRequests()),
"each move should delete the source volume")
// Verify delete requests reference the expected volume IDs.
deletedVols := make(map[uint32]bool)
for _, req := range source.DeleteRequests() {
deletedVols[req.VolumeId] = true
}
for _, vid := range volumeIDs {
require.True(t, deletedVols[vid], "volume %d should have been deleted from source", vid)
}
// Pre-delete verification should have called ReadVolumeFileStatus on both
// source and target for each volume.
require.Equal(t, len(volumeIDs), source.ReadFileStatusCount(),
"each move should read source volume status before delete")
require.Equal(t, len(volumeIDs), target.ReadFileStatusCount(),
"each move should read target volume status before delete")
// Target should have received copy and tail calls for all 3 volumes.
copyCalls, _, tailCalls := target.BalanceStats()
require.Equal(t, len(volumeIDs), copyCalls, "target should receive one copy per volume")
require.Equal(t, len(volumeIDs), tailCalls, "target should receive one tail per volume")
}
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.25-alpine AS builder
FROM golang:1.24-alpine AS builder
# Set working directory
WORKDIR /app
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.25-alpine AS builder
FROM golang:1.24-alpine AS builder
# Set working directory
WORKDIR /app
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.25-alpine AS builder
FROM golang:1.24-alpine AS builder
# Install git and other build dependencies
RUN apk add --no-cache git make
+4 -4
View File
@@ -140,7 +140,7 @@ func TestCORSConfigurationManagement(t *testing.T) {
Bucket: aws.String(bucketName),
CORSConfiguration: corsConfig,
})
require.NoError(t, err, "Should be able to put CORS configuration")
assert.NoError(t, err, "Should be able to put CORS configuration")
// Wait for metadata subscription to update cache
time.Sleep(50 * time.Millisecond)
@@ -149,9 +149,9 @@ func TestCORSConfigurationManagement(t *testing.T) {
getResp, err := client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err, "Should be able to get CORS configuration")
require.NotNil(t, getResp.CORSRules, "CORS configuration should not be nil")
require.Len(t, getResp.CORSRules, 1, "Should have one CORS rule")
assert.NoError(t, err, "Should be able to get CORS configuration")
assert.NotNil(t, getResp.CORSRules, "CORS configuration should not be nil")
assert.Len(t, getResp.CORSRules, 1, "Should have one CORS rule")
rule := getResp.CORSRules[0]
assert.Equal(t, []string{"*"}, rule.AllowedHeaders, "Allowed headers should match")
+1 -1
View File
@@ -1,5 +1,5 @@
# Multi-stage build for SeaweedFS S3 with IAM
FROM golang:1.25-alpine AS builder
FROM golang:1.23-alpine AS builder
# Install build dependencies
RUN apk add --no-cache git make curl wget
+1 -4
View File
@@ -185,9 +185,6 @@ test-context: ## Test only contextual policy enforcement
test-presigned: ## Test only presigned URL integration
go test -v -run TestS3IAMPresignedURLIntegration ./...
test-group: ## Run IAM group management tests
go test -v -run "TestIAMGroup" ./...
test-sts: ## Run all STS tests
go test -v -run "TestSTS" ./...
@@ -266,7 +263,7 @@ docker-build: ## Build custom SeaweedFS image for Docker tests
# All PHONY targets
.PHONY: test test-quick run-tests setup start-services stop-services wait-for-services clean logs status debug
.PHONY: test-auth test-policy test-expiration test-multipart test-bucket-policy test-context test-presigned test-group test-sts test-sts-assume-role test-sts-ldap
.PHONY: test-auth test-policy test-expiration test-multipart test-bucket-policy test-context test-presigned test-sts test-sts-assume-role test-sts-ldap
.PHONY: benchmark ci watch install-deps docker-test docker-up docker-down docker-logs docker-build
.PHONY: test-distributed test-performance test-stress test-versioning-stress test-keycloak-full test-all-previously-skipped setup-all-tests help-advanced
-792
View File
@@ -1,792 +0,0 @@
package iam
import (
"encoding/xml"
"io"
"net/http"
"net/url"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/iam"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestIAMGroupLifecycle tests the full lifecycle of group management:
// CreateGroup, GetGroup, ListGroups, DeleteGroup
func TestIAMGroupLifecycle(t *testing.T) {
framework := NewS3IAMTestFramework(t)
defer framework.Cleanup()
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
require.NoError(t, err)
groupName := "test-group-lifecycle"
t.Run("create_group", func(t *testing.T) {
resp, err := iamClient.CreateGroup(&iam.CreateGroupInput{
GroupName: aws.String(groupName),
})
require.NoError(t, err)
assert.Equal(t, groupName, *resp.Group.GroupName)
})
t.Run("get_group", func(t *testing.T) {
resp, err := iamClient.GetGroup(&iam.GetGroupInput{
GroupName: aws.String(groupName),
})
require.NoError(t, err)
assert.Equal(t, groupName, *resp.Group.GroupName)
})
t.Run("list_groups_contains_created", func(t *testing.T) {
resp, err := iamClient.ListGroups(&iam.ListGroupsInput{})
require.NoError(t, err)
found := false
for _, g := range resp.Groups {
if *g.GroupName == groupName {
found = true
break
}
}
assert.True(t, found, "Created group should appear in ListGroups")
})
t.Run("create_duplicate_group_fails", func(t *testing.T) {
_, err := iamClient.CreateGroup(&iam.CreateGroupInput{
GroupName: aws.String(groupName),
})
assert.Error(t, err, "Creating a duplicate group should fail")
})
t.Run("delete_group", func(t *testing.T) {
_, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{
GroupName: aws.String(groupName),
})
require.NoError(t, err)
// Verify it's gone
resp, err := iamClient.ListGroups(&iam.ListGroupsInput{})
require.NoError(t, err)
for _, g := range resp.Groups {
assert.NotEqual(t, groupName, *g.GroupName,
"Deleted group should not appear in ListGroups")
}
})
t.Run("delete_nonexistent_group_fails", func(t *testing.T) {
_, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{
GroupName: aws.String("nonexistent-group-xyz"),
})
assert.Error(t, err)
})
}
// TestIAMGroupMembership tests adding and removing users from groups
func TestIAMGroupMembership(t *testing.T) {
framework := NewS3IAMTestFramework(t)
defer framework.Cleanup()
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
require.NoError(t, err)
groupName := "test-group-members"
userName := "test-user-for-group"
// Setup: create group and user
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
GroupName: aws.String(groupName),
})
require.NoError(t, err)
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
_, err = iamClient.CreateUser(&iam.CreateUserInput{
UserName: aws.String(userName),
})
require.NoError(t, err)
defer iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
t.Run("add_user_to_group", func(t *testing.T) {
_, err := iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
GroupName: aws.String(groupName),
UserName: aws.String(userName),
})
require.NoError(t, err)
})
t.Run("get_group_shows_member", func(t *testing.T) {
resp, err := iamClient.GetGroup(&iam.GetGroupInput{
GroupName: aws.String(groupName),
})
require.NoError(t, err)
found := false
for _, u := range resp.Users {
if *u.UserName == userName {
found = true
break
}
}
assert.True(t, found, "Added user should appear in GetGroup members")
})
t.Run("list_groups_for_user", func(t *testing.T) {
resp, err := iamClient.ListGroupsForUser(&iam.ListGroupsForUserInput{
UserName: aws.String(userName),
})
require.NoError(t, err)
found := false
for _, g := range resp.Groups {
if *g.GroupName == groupName {
found = true
break
}
}
assert.True(t, found, "Group should appear in ListGroupsForUser")
})
t.Run("add_duplicate_member_is_idempotent", func(t *testing.T) {
_, err := iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
GroupName: aws.String(groupName),
UserName: aws.String(userName),
})
// Should succeed (idempotent) or return a benign error
// AWS IAM allows duplicate add without error
assert.NoError(t, err)
})
t.Run("remove_user_from_group", func(t *testing.T) {
_, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
GroupName: aws.String(groupName),
UserName: aws.String(userName),
})
require.NoError(t, err)
// Verify removal
resp, err := iamClient.GetGroup(&iam.GetGroupInput{
GroupName: aws.String(groupName),
})
require.NoError(t, err)
for _, u := range resp.Users {
assert.NotEqual(t, userName, *u.UserName,
"Removed user should not appear in group members")
}
})
}
// TestIAMGroupPolicyAttachment tests attaching and detaching policies from groups
func TestIAMGroupPolicyAttachment(t *testing.T) {
framework := NewS3IAMTestFramework(t)
defer framework.Cleanup()
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
require.NoError(t, err)
groupName := "test-group-policies"
policyName := "test-group-attach-policy"
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"*"}]}`
// Setup: create group and policy
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
GroupName: aws.String(groupName),
})
require.NoError(t, err)
createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
PolicyName: aws.String(policyName),
PolicyDocument: aws.String(policyDoc),
})
require.NoError(t, err)
policyArn := createPolicyResp.Policy.Arn
// Cleanup in correct order: detach policy, delete group, delete policy
t.Cleanup(func() {
if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
GroupName: aws.String(groupName),
PolicyArn: policyArn,
}); err != nil {
t.Logf("cleanup: failed to detach group policy: %v", err)
}
if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
t.Logf("cleanup: failed to delete group: %v", err)
}
if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn}); err != nil {
t.Logf("cleanup: failed to delete policy: %v", err)
}
})
t.Run("attach_group_policy", func(t *testing.T) {
_, err := iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
GroupName: aws.String(groupName),
PolicyArn: policyArn,
})
require.NoError(t, err)
})
t.Run("list_attached_group_policies", func(t *testing.T) {
resp, err := iamClient.ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{
GroupName: aws.String(groupName),
})
require.NoError(t, err)
found := false
for _, p := range resp.AttachedPolicies {
if *p.PolicyName == policyName {
found = true
break
}
}
assert.True(t, found, "Attached policy should appear in ListAttachedGroupPolicies")
})
t.Run("detach_group_policy", func(t *testing.T) {
_, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
GroupName: aws.String(groupName),
PolicyArn: policyArn,
})
require.NoError(t, err)
// Verify detachment
resp, err := iamClient.ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{
GroupName: aws.String(groupName),
})
require.NoError(t, err)
for _, p := range resp.AttachedPolicies {
assert.NotEqual(t, policyName, *p.PolicyName,
"Detached policy should not appear in ListAttachedGroupPolicies")
}
})
}
// TestIAMGroupPolicyEnforcement tests that group policies are enforced during S3 operations.
// Creates a user with no direct policies, adds them to a group with S3 access,
// and verifies they can access S3 through the group policy.
func TestIAMGroupPolicyEnforcement(t *testing.T) {
framework := NewS3IAMTestFramework(t)
defer framework.Cleanup()
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
require.NoError(t, err)
groupName := "test-enforcement-group"
userName := "test-enforcement-user"
policyName := "test-enforcement-policy"
bucketName := "test-group-enforce-bucket"
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"]}]}`
// Create user
_, err = iamClient.CreateUser(&iam.CreateUserInput{
UserName: aws.String(userName),
})
require.NoError(t, err)
// Create access key for the user
keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{
UserName: aws.String(userName),
})
require.NoError(t, err)
accessKeyId := *keyResp.AccessKey.AccessKeyId
secretKey := *keyResp.AccessKey.SecretAccessKey
// Create an S3 client with the user's credentials
userS3Client := createS3Client(t, accessKeyId, secretKey)
// Create group
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
GroupName: aws.String(groupName),
})
require.NoError(t, err)
// Create policy
createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
PolicyName: aws.String(policyName),
PolicyDocument: aws.String(policyDoc),
})
require.NoError(t, err)
policyArn := createPolicyResp.Policy.Arn
// Cleanup in correct order: remove user from group, detach policy,
// delete access key, delete user, delete group, delete policy
t.Cleanup(func() {
if _, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
GroupName: aws.String(groupName),
UserName: aws.String(userName),
}); err != nil {
t.Logf("cleanup: failed to remove user from group: %v", err)
}
if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
GroupName: aws.String(groupName),
PolicyArn: policyArn,
}); err != nil {
t.Logf("cleanup: failed to detach group policy: %v", err)
}
if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
UserName: aws.String(userName),
AccessKeyId: keyResp.AccessKey.AccessKeyId,
}); err != nil {
t.Logf("cleanup: failed to delete access key: %v", err)
}
if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
t.Logf("cleanup: failed to delete user: %v", err)
}
if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
t.Logf("cleanup: failed to delete group: %v", err)
}
if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn}); err != nil {
t.Logf("cleanup: failed to delete policy: %v", err)
}
})
// Register bucket cleanup on parent test with admin credentials
// (userS3Client may lack permissions by cleanup time)
adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
require.NoError(t, err)
t.Cleanup(func() {
if _, err := adminS3.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String("test-key"),
}); err != nil {
t.Logf("cleanup: failed to delete object: %v", err)
}
if _, err := adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)}); err != nil {
t.Logf("cleanup: failed to delete bucket: %v", err)
}
})
t.Run("user_without_group_denied", func(t *testing.T) {
// User has no policies and is not in any group — should be denied
_, err := userS3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
require.Error(t, err, "User without any policies should be denied")
awsErr, ok := err.(awserr.Error)
require.True(t, ok, "Expected awserr.Error")
assert.Equal(t, "AccessDenied", awsErr.Code())
})
t.Run("user_with_group_policy_allowed", func(t *testing.T) {
// Attach policy to group
_, err := iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
GroupName: aws.String(groupName),
PolicyArn: policyArn,
})
require.NoError(t, err)
// Add user to group
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
GroupName: aws.String(groupName),
UserName: aws.String(userName),
})
require.NoError(t, err)
// Wait for policy propagation, then create bucket
require.Eventually(t, func() bool {
_, err = userS3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
return err == nil
}, 10*time.Second, 500*time.Millisecond, "User with group policy should be allowed")
// Should also be able to put/get objects
_, err = userS3Client.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String("test-key"),
Body: aws.ReadSeekCloser(strings.NewReader("test-data")),
})
require.NoError(t, err, "User should be able to put objects through group policy")
})
t.Run("user_removed_from_group_denied", func(t *testing.T) {
// Remove user from group
_, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
GroupName: aws.String(groupName),
UserName: aws.String(userName),
})
require.NoError(t, err)
// Wait for policy propagation — user should now be denied
var lastErr error
require.Eventually(t, func() bool {
_, lastErr = userS3Client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(bucketName),
})
return lastErr != nil
}, 10*time.Second, 500*time.Millisecond, "User removed from group should be denied")
awsErr, ok := lastErr.(awserr.Error)
require.True(t, ok, "Expected awserr.Error")
assert.Equal(t, "AccessDenied", awsErr.Code())
})
}
// TestIAMGroupDisabledPolicyEnforcement tests that disabled groups do not contribute policies.
// Uses the raw IAM API (callIAMAPI) since the AWS SDK doesn't support custom group status.
func TestIAMGroupDisabledPolicyEnforcement(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !isSeaweedFSRunning(t) {
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
}
framework := NewS3IAMTestFramework(t)
defer framework.Cleanup()
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
require.NoError(t, err)
groupName := "test-disabled-group"
userName := "test-disabled-grp-user"
policyName := "test-disabled-grp-policy"
bucketName := "test-disabled-grp-bucket"
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"]}]}`
// Create user, group, policy
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
require.NoError(t, err)
keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{UserName: aws.String(userName)})
require.NoError(t, err)
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(groupName)})
require.NoError(t, err)
createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
PolicyName: aws.String(policyName), PolicyDocument: aws.String(policyDoc),
})
require.NoError(t, err)
// Cleanup in correct order: remove user from group, detach policy,
// delete access key, delete user, delete group, delete policy
t.Cleanup(func() {
if _, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
GroupName: aws.String(groupName), UserName: aws.String(userName),
}); err != nil {
t.Logf("cleanup: failed to remove user from group: %v", err)
}
if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
GroupName: aws.String(groupName),
PolicyArn: aws.String("arn:aws:iam:::policy/" + policyName),
}); err != nil {
t.Logf("cleanup: failed to detach group policy: %v", err)
}
if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
UserName: aws.String(userName), AccessKeyId: keyResp.AccessKey.AccessKeyId,
}); err != nil {
t.Logf("cleanup: failed to delete access key: %v", err)
}
if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
t.Logf("cleanup: failed to delete user: %v", err)
}
if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
t.Logf("cleanup: failed to delete group: %v", err)
}
if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: createPolicyResp.Policy.Arn}); err != nil {
t.Logf("cleanup: failed to delete policy: %v", err)
}
})
// Setup: attach policy, add user, create bucket with admin
_, err = iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
GroupName: aws.String(groupName), PolicyArn: createPolicyResp.Policy.Arn,
})
require.NoError(t, err)
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
GroupName: aws.String(groupName), UserName: aws.String(userName),
})
require.NoError(t, err)
userS3Client := createS3Client(t, *keyResp.AccessKey.AccessKeyId, *keyResp.AccessKey.SecretAccessKey)
// Create bucket using admin first so we can test listing
adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
require.NoError(t, err)
_, err = adminS3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucketName)})
require.NoError(t, err)
defer adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
t.Run("enabled_group_allows_access", func(t *testing.T) {
require.Eventually(t, func() bool {
_, err := userS3Client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(bucketName),
})
return err == nil
}, 10*time.Second, 500*time.Millisecond, "User in enabled group should have access")
})
t.Run("disabled_group_denies_access", func(t *testing.T) {
// Disable group via raw IAM API (no SDK support for this extension)
resp, err := callIAMAPIAuthenticated(t, framework, "UpdateGroup", url.Values{
"GroupName": {groupName},
"Disabled": {"true"},
})
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode, "UpdateGroup (disable) should return 200")
// Wait for propagation — user should be denied
var lastErr error
require.Eventually(t, func() bool {
_, lastErr = userS3Client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(bucketName),
})
return lastErr != nil
}, 10*time.Second, 500*time.Millisecond, "User in disabled group should be denied access")
awsErr, ok := lastErr.(awserr.Error)
require.True(t, ok, "Expected awserr.Error")
assert.Equal(t, "AccessDenied", awsErr.Code())
})
t.Run("re_enabled_group_restores_access", func(t *testing.T) {
// Re-enable the group
resp, err := callIAMAPIAuthenticated(t, framework, "UpdateGroup", url.Values{
"GroupName": {groupName},
"Disabled": {"false"},
})
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode, "UpdateGroup (re-enable) should return 200")
// Wait for propagation — user should have access again
require.Eventually(t, func() bool {
_, err = userS3Client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(bucketName),
})
return err == nil
}, 10*time.Second, 500*time.Millisecond, "User in re-enabled group should have access again")
})
}
// TestIAMGroupUserDeletionSideEffect tests that deleting a user removes them from all groups.
func TestIAMGroupUserDeletionSideEffect(t *testing.T) {
framework := NewS3IAMTestFramework(t)
defer framework.Cleanup()
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
require.NoError(t, err)
groupName := "test-deletion-group"
userName := "test-deletion-user"
// Create group and user
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(groupName)})
require.NoError(t, err)
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
require.NoError(t, err)
t.Cleanup(func() {
// Best-effort: user may already be deleted by the test
iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
})
// Add user to group
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
GroupName: aws.String(groupName),
UserName: aws.String(userName),
})
require.NoError(t, err)
// Verify user is in group
getResp, err := iamClient.GetGroup(&iam.GetGroupInput{GroupName: aws.String(groupName)})
require.NoError(t, err)
assert.Len(t, getResp.Users, 1, "Group should have 1 member before deletion")
// Delete the user
_, err = iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
require.NoError(t, err)
// Verify user was removed from the group
getResp, err = iamClient.GetGroup(&iam.GetGroupInput{GroupName: aws.String(groupName)})
require.NoError(t, err)
assert.Empty(t, getResp.Users, "Group should have no members after user deletion")
}
// TestIAMGroupMultipleGroups tests that a user can belong to multiple groups
// and inherits policies from all of them.
func TestIAMGroupMultipleGroups(t *testing.T) {
framework := NewS3IAMTestFramework(t)
defer framework.Cleanup()
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
require.NoError(t, err)
group1 := "test-multi-group-1"
group2 := "test-multi-group-2"
userName := "test-multi-group-user"
// Create two groups
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(group1)})
require.NoError(t, err)
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(group1)})
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(group2)})
require.NoError(t, err)
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(group2)})
// Create user
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
require.NoError(t, err)
defer func() {
iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
GroupName: aws.String(group1), UserName: aws.String(userName),
})
iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
GroupName: aws.String(group2), UserName: aws.String(userName),
})
iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
}()
// Add user to both groups
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
GroupName: aws.String(group1), UserName: aws.String(userName),
})
require.NoError(t, err)
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
GroupName: aws.String(group2), UserName: aws.String(userName),
})
require.NoError(t, err)
// Verify user appears in both groups
resp, err := iamClient.ListGroupsForUser(&iam.ListGroupsForUserInput{
UserName: aws.String(userName),
})
require.NoError(t, err)
groupNames := make(map[string]bool)
for _, g := range resp.Groups {
groupNames[*g.GroupName] = true
}
assert.True(t, groupNames[group1], "User should be in group 1")
assert.True(t, groupNames[group2], "User should be in group 2")
}
// --- Response types for raw IAM API calls ---
type CreateGroupResponse struct {
XMLName xml.Name `xml:"CreateGroupResponse"`
CreateGroupResult struct {
Group struct {
GroupName string `xml:"GroupName"`
} `xml:"Group"`
} `xml:"CreateGroupResult"`
}
type ListGroupsResponse struct {
XMLName xml.Name `xml:"ListGroupsResponse"`
ListGroupsResult struct {
Groups []struct {
GroupName string `xml:"GroupName"`
} `xml:"Groups>member"`
} `xml:"ListGroupsResult"`
}
// callIAMAPIAuthenticated sends an authenticated raw IAM API request using the
// framework's JWT token. This is needed for custom extensions not in the AWS SDK
// (like UpdateGroup with Disabled parameter).
func callIAMAPIAuthenticated(_ *testing.T, framework *S3IAMTestFramework, action string, params url.Values) (*http.Response, error) {
params.Set("Action", action)
req, err := http.NewRequest(http.MethodPost, TestIAMEndpoint+"/",
strings.NewReader(params.Encode()))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
token, err := framework.generateSTSSessionToken("admin-user", "TestAdminRole", time.Hour, "", nil)
if err != nil {
return nil, err
}
client := &http.Client{
Timeout: 30 * time.Second,
Transport: &BearerTokenTransport{Token: token},
}
return client.Do(req)
}
// TestIAMGroupRawAPI tests group operations using raw HTTP IAM API calls,
// verifying XML response format for group operations.
func TestIAMGroupRawAPI(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !isSeaweedFSRunning(t) {
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
}
framework := NewS3IAMTestFramework(t)
defer framework.Cleanup()
groupName := "test-raw-api-group"
t.Run("create_group_raw", func(t *testing.T) {
resp, err := callIAMAPIAuthenticated(t, framework, "CreateGroup", url.Values{
"GroupName": {groupName},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
var createResp CreateGroupResponse
err = xml.Unmarshal(body, &createResp)
require.NoError(t, err)
assert.Equal(t, groupName, createResp.CreateGroupResult.Group.GroupName)
})
t.Run("list_groups_raw", func(t *testing.T) {
resp, err := callIAMAPIAuthenticated(t, framework, "ListGroups", url.Values{})
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
var listResp ListGroupsResponse
err = xml.Unmarshal(body, &listResp)
require.NoError(t, err)
found := false
for _, g := range listResp.ListGroupsResult.Groups {
if g.GroupName == groupName {
found = true
break
}
}
assert.True(t, found, "Created group should appear in raw ListGroups")
})
t.Run("delete_group_raw", func(t *testing.T) {
resp, err := callIAMAPIAuthenticated(t, framework, "DeleteGroup", url.Values{
"GroupName": {groupName},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
})
}
// createS3Client creates an S3 client with static credentials
func createS3Client(t *testing.T, accessKey, secretKey string) *s3.S3 {
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(TestS3Endpoint),
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
})
require.NoError(t, err)
return s3.New(sess)
}
@@ -1,573 +0,0 @@
package example
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
v1credentials "github.com/aws/aws-sdk-go/aws/credentials"
v1signer "github.com/aws/aws-sdk-go/aws/signer/v4"
v1s3 "github.com/aws/aws-sdk-go/service/s3"
v2aws "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
v2s3 "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// newS3V2Client creates an AWS SDK v2 S3 client from the test cluster.
func newS3V2Client(cluster *TestCluster) *v2s3.Client {
return v2s3.New(v2s3.Options{
Region: testRegion,
BaseEndpoint: v2aws.String(cluster.s3Endpoint),
Credentials: v2aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, "")),
UsePathStyle: true,
})
}
func TestGetObjectAttributes(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
cluster, err := startMiniCluster(t)
require.NoError(t, err)
defer cluster.Stop()
t.Run("Basic", func(t *testing.T) {
testGetObjectAttributesBasic(t, cluster)
})
t.Run("MultipartObject", func(t *testing.T) {
testGetObjectAttributesMultipart(t, cluster)
})
t.Run("SelectiveAttributes", func(t *testing.T) {
testGetObjectAttributesSelective(t, cluster)
})
t.Run("InvalidAttribute", func(t *testing.T) {
testGetObjectAttributesInvalid(t, cluster)
})
t.Run("NonExistentObject", func(t *testing.T) {
testGetObjectAttributesNotFound(t, cluster)
})
t.Run("VersionedObject", func(t *testing.T) {
testGetObjectAttributesVersioned(t, cluster)
})
t.Run("ConditionalHeaders", func(t *testing.T) {
testGetObjectAttributesConditionalHeaders(t, cluster)
})
t.Run("VersionedConditionalHeaders", func(t *testing.T) {
testGetObjectAttributesVersionedConditionalHeaders(t, cluster)
})
}
func testGetObjectAttributesBasic(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-basic-")
objectKey := "test-object.txt"
objectData := "Hello, GetObjectAttributes!"
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader([]byte(objectData)),
})
require.NoError(t, err)
client := newS3V2Client(cluster)
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String(objectKey),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesEtag,
types.ObjectAttributesStorageClass,
types.ObjectAttributesObjectSize,
types.ObjectAttributesObjectParts,
},
})
require.NoError(t, err)
// ETag should be present and non-empty
require.NotNil(t, resp.ETag)
assert.NotEmpty(t, *resp.ETag)
assert.False(t, strings.Contains(*resp.ETag, `"`), "ETag in XML body should not have quotes")
// ObjectSize should match
require.NotNil(t, resp.ObjectSize)
assert.Equal(t, int64(len(objectData)), *resp.ObjectSize)
// StorageClass should be STANDARD (default)
assert.Equal(t, "STANDARD", string(resp.StorageClass))
// ObjectParts should be nil for non-multipart objects
assert.Nil(t, resp.ObjectParts)
// LastModified header should be present
assert.NotNil(t, resp.LastModified)
t.Logf("Basic GetObjectAttributes passed: ETag=%s, Size=%d, StorageClass=%s",
*resp.ETag, *resp.ObjectSize, resp.StorageClass)
}
func testGetObjectAttributesMultipart(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-mp-")
objectKey := "test-multipart.bin"
// Create a 2-part multipart upload
part1Data := bytes.Repeat([]byte("A"), 5*1024*1024) // 5MB (minimum part size)
part2Data := bytes.Repeat([]byte("B"), 3*1024*1024) // 3MB
initResp, err := cluster.s3Client.CreateMultipartUpload(&v1s3.CreateMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err)
uploadID := initResp.UploadId
part1Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
PartNumber: aws.Int64(1),
UploadId: uploadID,
Body: bytes.NewReader(part1Data),
})
require.NoError(t, err)
part2Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
PartNumber: aws.Int64(2),
UploadId: uploadID,
Body: bytes.NewReader(part2Data),
})
require.NoError(t, err)
_, err = cluster.s3Client.CompleteMultipartUpload(&v1s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: uploadID,
MultipartUpload: &v1s3.CompletedMultipartUpload{
Parts: []*v1s3.CompletedPart{
{ETag: part1Resp.ETag, PartNumber: aws.Int64(1)},
{ETag: part2Resp.ETag, PartNumber: aws.Int64(2)},
},
},
})
require.NoError(t, err)
// Wait briefly for metadata to settle
time.Sleep(200 * time.Millisecond)
client := newS3V2Client(cluster)
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String(objectKey),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesObjectParts,
types.ObjectAttributesObjectSize,
},
})
require.NoError(t, err)
require.NotNil(t, resp.ObjectSize)
assert.Equal(t, int64(len(part1Data)+len(part2Data)), *resp.ObjectSize)
require.NotNil(t, resp.ObjectParts, "ObjectParts should be present for multipart objects")
assert.Equal(t, int32(2), *resp.ObjectParts.TotalPartsCount)
require.Len(t, resp.ObjectParts.Parts, 2)
assert.Equal(t, int32(1), *resp.ObjectParts.Parts[0].PartNumber)
assert.Equal(t, int64(len(part1Data)), *resp.ObjectParts.Parts[0].Size)
assert.Equal(t, int32(2), *resp.ObjectParts.Parts[1].PartNumber)
assert.Equal(t, int64(len(part2Data)), *resp.ObjectParts.Parts[1].Size)
// Test pagination: MaxParts=1
resp2, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String(objectKey),
MaxParts: v2aws.Int32(1),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesObjectParts,
},
})
require.NoError(t, err)
require.NotNil(t, resp2.ObjectParts)
assert.Len(t, resp2.ObjectParts.Parts, 1)
assert.True(t, *resp2.ObjectParts.IsTruncated)
assert.Equal(t, int32(2), *resp2.ObjectParts.TotalPartsCount)
t.Logf("Multipart GetObjectAttributes passed: %d parts, total size %d",
*resp.ObjectParts.TotalPartsCount, *resp.ObjectSize)
}
func testGetObjectAttributesSelective(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-sel-")
objectKey := "test-selective.txt"
objectData := "Selective attributes test"
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader([]byte(objectData)),
})
require.NoError(t, err)
client := newS3V2Client(cluster)
// Request only ETag
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String(objectKey),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesEtag,
},
})
require.NoError(t, err)
require.NotNil(t, resp.ETag)
assert.NotEmpty(t, *resp.ETag)
assert.Nil(t, resp.ObjectSize, "ObjectSize should not be present when not requested")
assert.Empty(t, string(resp.StorageClass), "StorageClass should not be present when not requested")
assert.Nil(t, resp.ObjectParts, "ObjectParts should not be present when not requested")
t.Logf("Selective GetObjectAttributes passed: ETag=%s", *resp.ETag)
}
func testGetObjectAttributesInvalid(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-inv-")
objectKey := "test-object.txt"
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader([]byte("test")),
})
require.NoError(t, err)
// Use raw HTTP to send an invalid attribute name since the SDK validates
reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
req, err := http.NewRequest("GET", reqURL, nil)
require.NoError(t, err)
req.Header.Set("X-Amz-Object-Attributes", "InvalidAttr")
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
require.NoError(t, err)
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 400, resp.StatusCode)
t.Logf("Invalid attribute test passed: got %d", resp.StatusCode)
}
func testGetObjectAttributesNotFound(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-nf-")
client := newS3V2Client(cluster)
_, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("nonexistent-key"),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesEtag,
},
})
require.Error(t, err)
assert.Contains(t, err.Error(), "NoSuchKey")
t.Logf("NotFound GetObjectAttributes passed")
}
func testGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster) {
client := newS3V2Client(cluster)
bucketName := createTestBucket(t, cluster, "test-goa-ver-")
// Enable versioning
_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
Bucket: v2aws.String(bucketName),
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
require.NoError(t, err)
time.Sleep(200 * time.Millisecond)
// Put two versions of the same object
v1Data := "version 1 content"
putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("versioned-key"),
Body: strings.NewReader(v1Data),
})
require.NoError(t, err)
require.NotNil(t, putResp1.VersionId)
versionId1 := *putResp1.VersionId
v2Data := "version 2 content - longer"
putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("versioned-key"),
Body: strings.NewReader(v2Data),
})
require.NoError(t, err)
require.NotNil(t, putResp2.VersionId)
versionId2 := *putResp2.VersionId
assert.NotEqual(t, versionId1, versionId2, "versions should differ")
// GetObjectAttributes for latest version (v2)
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("versioned-key"),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesObjectSize,
types.ObjectAttributesEtag,
},
})
require.NoError(t, err)
require.NotNil(t, resp.ObjectSize)
assert.Equal(t, int64(len(v2Data)), *resp.ObjectSize)
require.NotNil(t, resp.VersionId)
assert.Equal(t, versionId2, *resp.VersionId)
// GetObjectAttributes for specific older version (v1)
resp1, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("versioned-key"),
VersionId: v2aws.String(versionId1),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesObjectSize,
types.ObjectAttributesEtag,
},
})
require.NoError(t, err)
require.NotNil(t, resp1.ObjectSize)
assert.Equal(t, int64(len(v1Data)), *resp1.ObjectSize)
require.NotNil(t, resp1.VersionId)
assert.Equal(t, versionId1, *resp1.VersionId)
t.Logf("Versioned GetObjectAttributes passed: v1 size=%d (id=%s), v2 size=%d (id=%s)",
*resp1.ObjectSize, versionId1, *resp.ObjectSize, versionId2)
}
// signedGetObjectAttributes creates a signed GET request for ?attributes with custom headers.
func signedGetObjectAttributes(t *testing.T, cluster *TestCluster, bucketName, objectKey string, extraHeaders map[string]string) *http.Response {
reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
req, err := http.NewRequest("GET", reqURL, nil)
require.NoError(t, err)
req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
for k, v := range extraHeaders {
req.Header.Set(k, v)
}
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
require.NoError(t, err)
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
require.NoError(t, err)
return resp
}
func testGetObjectAttributesConditionalHeaders(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-cond-")
objectKey := "cond-test.txt"
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader([]byte("conditional headers test")),
})
require.NoError(t, err)
// Get the ETag and Last-Modified for the object
headResp, err := cluster.s3Client.HeadObject(&v1s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err)
etag := aws.StringValue(headResp.ETag)
lastModified := headResp.LastModified
require.NotNil(t, lastModified)
pastDate := lastModified.Add(-1 * time.Hour).UTC().Format(http.TimeFormat)
futureDate := lastModified.Add(1 * time.Hour).UTC().Format(http.TimeFormat)
// RFC 7232: If-Match true + If-Unmodified-Since false => 200 OK
// If-Unmodified-Since is ignored when If-Match is present
t.Run("IfMatch_true_IfUnmodifiedSince_false", func(t *testing.T) {
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
"If-Match": etag,
"If-Unmodified-Since": pastDate, // object was modified after this => false
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 200, resp.StatusCode,
"If-Match=true should return 200 even when If-Unmodified-Since=false (RFC 7232 Section 3.4)")
})
// RFC 7232: If-None-Match false + If-Modified-Since true => 304 Not Modified
// If-Modified-Since is ignored when If-None-Match is present
t.Run("IfNoneMatch_false_IfModifiedSince_true", func(t *testing.T) {
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
"If-None-Match": etag,
"If-Modified-Since": pastDate, // object was modified after this => true
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 304, resp.StatusCode,
"If-None-Match=false (ETag match) should return 304 even when If-Modified-Since=true (RFC 7232 Section 3.3)")
})
// If-Match succeeds, If-Unmodified-Since also succeeds => 200
t.Run("IfMatch_true_IfUnmodifiedSince_true", func(t *testing.T) {
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
"If-Match": etag,
"If-Unmodified-Since": futureDate,
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 200, resp.StatusCode)
})
// If-None-Match passes (ETag differs), If-Modified-Since ignored => 200
// Per RFC 7232, If-Modified-Since is ignored when If-None-Match is present
t.Run("IfNoneMatch_true_IfModifiedSince_ignored", func(t *testing.T) {
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
"If-None-Match": `"nonexistent-etag"`,
"If-Modified-Since": futureDate, // would fail alone, but is ignored
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 200, resp.StatusCode,
"If-None-Match=true means If-Modified-Since is ignored, should return 200 (RFC 7232 Section 3.3)")
})
// If-Match fails => 412 regardless of If-Unmodified-Since
t.Run("IfMatch_false", func(t *testing.T) {
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
"If-Match": `"wrong-etag"`,
"If-Unmodified-Since": futureDate,
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 412, resp.StatusCode)
})
t.Logf("Conditional headers tests passed")
}
// signedGetObjectAttributesVersioned creates a signed GET request for ?attributes&versionId=... with custom headers.
func signedGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster, bucketName, objectKey, versionId string, extraHeaders map[string]string) *http.Response {
reqURL := fmt.Sprintf("%s/%s/%s?attributes&versionId=%s", cluster.s3Endpoint, bucketName, objectKey, versionId)
req, err := http.NewRequest("GET", reqURL, nil)
require.NoError(t, err)
req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
for k, v := range extraHeaders {
req.Header.Set(k, v)
}
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
require.NoError(t, err)
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
require.NoError(t, err)
return resp
}
func testGetObjectAttributesVersionedConditionalHeaders(t *testing.T, cluster *TestCluster) {
client := newS3V2Client(cluster)
bucketName := createTestBucket(t, cluster, "test-goa-vcond-")
// Enable versioning
_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
Bucket: v2aws.String(bucketName),
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
require.NoError(t, err)
time.Sleep(200 * time.Millisecond)
// Put two versions with different content (different ETags)
v1Data := "version 1 - original"
putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("vcond-key"),
Body: strings.NewReader(v1Data),
})
require.NoError(t, err)
require.NotNil(t, putResp1.VersionId)
vid1 := *putResp1.VersionId
v2Data := "version 2 - updated content"
putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("vcond-key"),
Body: strings.NewReader(v2Data),
})
require.NoError(t, err)
require.NotNil(t, putResp2.VersionId)
vid2 := *putResp2.VersionId
// Get ETags for each version
headV1, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("vcond-key"),
VersionId: v2aws.String(vid1),
})
require.NoError(t, err)
etagV1 := *headV1.ETag
headV2, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("vcond-key"),
VersionId: v2aws.String(vid2),
})
require.NoError(t, err)
etagV2 := *headV2.ETag
require.NotEqual(t, etagV1, etagV2, "versions should have different ETags")
// If-Match with v1's ETag + versionId=v1 => 200
// Before the fix, this would fail with 412 because conditional headers
// were evaluated against the latest version (v2) whose ETag differs
t.Run("IfMatch_v1_etag_versionId_v1", func(t *testing.T) {
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
"If-Match": etagV1,
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 200, resp.StatusCode,
"If-Match with v1 ETag targeting versionId=v1 should return 200")
})
// If-Match with v2's ETag + versionId=v1 => 412
// The ETag doesn't match v1, so this should fail
t.Run("IfMatch_v2_etag_versionId_v1", func(t *testing.T) {
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
"If-Match": etagV2,
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 412, resp.StatusCode,
"If-Match with v2 ETag targeting versionId=v1 should return 412")
})
// If-None-Match with v1's ETag + versionId=v1 => 304
t.Run("IfNoneMatch_v1_etag_versionId_v1", func(t *testing.T) {
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
"If-None-Match": etagV1,
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 304, resp.StatusCode,
"If-None-Match with v1 ETag targeting versionId=v1 should return 304")
})
t.Logf("Versioned conditional headers tests passed: vid1=%s, vid2=%s", vid1, vid2)
}
-212
View File
@@ -422,218 +422,6 @@ func TestS3MultipartOperationsInheritPutObjectPermissions(t *testing.T) {
require.Equal(t, 0, len(listUploadsOut.Uploads))
}
// TestS3IAMManagedPolicyLifecycle is an end-to-end integration test covering the
// user-reported use case in https://github.com/seaweedfs/seaweedfs/issues/8506
// where managed policy operations (GetPolicy, ListPolicies, DeletePolicy,
// AttachUserPolicy, DetachUserPolicy) returned 500 errors.
func TestS3IAMManagedPolicyLifecycle(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
cluster, err := startMiniCluster(t)
require.NoError(t, err)
defer cluster.Stop()
iamClient := newIAMClient(t, cluster.s3Endpoint)
// Step 1: Create a user (this already worked per the issue)
userName := uniqueName("lifecycle-user")
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
require.NoError(t, err, "CreateUser should succeed")
// Step 2: Create a managed policy via IAM API
policyName := uniqueName("lifecycle-policy")
policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
policyDoc := `{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject"],
"Resource": "arn:aws:s3:::*"
}]
}`
createOut, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
PolicyName: aws.String(policyName),
PolicyDocument: aws.String(policyDoc),
})
require.NoError(t, err, "CreatePolicy should succeed")
require.NotNil(t, createOut.Policy)
require.Equal(t, policyName, *createOut.Policy.PolicyName)
// Step 3: ListPolicies — should include the created policy (was returning 500)
listOut, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
require.NoError(t, err, "ListPolicies should succeed (was returning 500)")
require.True(t, managedPolicyContains(listOut.Policies, policyName),
"ListPolicies should contain the newly created policy")
// Step 4: GetPolicy by ARN — should return the policy (was returning 500)
getOut, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
require.NoError(t, err, "GetPolicy should succeed (was returning 500)")
require.NotNil(t, getOut.Policy)
require.Equal(t, policyName, *getOut.Policy.PolicyName)
require.Equal(t, policyArn, *getOut.Policy.Arn)
// Step 5: AttachUserPolicy — should succeed (was returning 500)
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String(policyArn),
})
require.NoError(t, err, "AttachUserPolicy should succeed (was returning 500)")
// Step 6: ListAttachedUserPolicies — verify the policy is attached
attachedOut, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
UserName: aws.String(userName),
})
require.NoError(t, err, "ListAttachedUserPolicies should succeed")
require.True(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
"Policy should appear in user's attached policies")
// Step 7: Idempotent re-attach should not fail
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String(policyArn),
})
require.NoError(t, err, "Re-attaching same policy should be idempotent")
// Step 8: DeletePolicy while attached — should fail with DeleteConflict (AWS behavior)
_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
require.Error(t, err, "DeletePolicy should fail while policy is attached")
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeDeleteConflictException, awsErr.Code(),
"Should return DeleteConflict when deleting attached policy")
// Step 9: DetachUserPolicy
_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String(policyArn),
})
require.NoError(t, err, "DetachUserPolicy should succeed")
// Verify detached
attachedOut, err = iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
UserName: aws.String(userName),
})
require.NoError(t, err)
require.False(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
"Policy should no longer appear in user's attached policies after detach")
// Step 10: DeletePolicy — should now succeed (was returning XML parsing error)
_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
require.NoError(t, err, "DeletePolicy should succeed after detach (was returning XML parsing error)")
// Step 11: Verify the policy is gone
listOut, err = iamClient.ListPolicies(&iam.ListPoliciesInput{})
require.NoError(t, err)
require.False(t, managedPolicyContains(listOut.Policies, policyName),
"Deleted policy should not appear in ListPolicies")
_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
require.Error(t, err, "GetPolicy should fail for deleted policy")
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
}
// TestS3IAMManagedPolicyErrorCases covers error cases from the user-reported issue:
// invalid ARNs, missing policies, and missing users.
func TestS3IAMManagedPolicyErrorCases(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
cluster, err := startMiniCluster(t)
require.NoError(t, err)
defer cluster.Stop()
iamClient := newIAMClient(t, cluster.s3Endpoint)
t.Run("GetPolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
_, err := iamClient.GetPolicy(&iam.GetPolicyInput{
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
t.Run("DeletePolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
_, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
t.Run("AttachUserPolicy with nonexistent policy returns NoSuchEntity", func(t *testing.T) {
userName := uniqueName("err-user")
_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
require.NoError(t, err)
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
t.Run("AttachUserPolicy with nonexistent user returns NoSuchEntity", func(t *testing.T) {
policyName := uniqueName("err-policy")
_, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
PolicyName: aws.String(policyName),
PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
})
require.NoError(t, err)
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
UserName: aws.String("nonexistent-user"),
PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
t.Run("DetachUserPolicy that is not attached returns NoSuchEntity", func(t *testing.T) {
userName := uniqueName("detach-user")
_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
require.NoError(t, err)
policyName := uniqueName("detach-policy")
_, err = iamClient.CreatePolicy(&iam.CreatePolicyInput{
PolicyName: aws.String(policyName),
PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
})
require.NoError(t, err)
_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
t.Run("ListAttachedUserPolicies for nonexistent user returns NoSuchEntity", func(t *testing.T) {
_, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
UserName: aws.String("nonexistent-user"),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
}
func execShell(t *testing.T, weedCmd, master, filer, shellCmd string) string {
// weed shell -master=... -filer=...
args := []string{"shell", "-master=" + master, "-filer=" + filer}
-12
View File
@@ -1,12 +0,0 @@
FROM alpine:3.20
RUN apk add --no-cache curl && \
addgroup -S seaweed && \
adduser -S seaweed -G seaweed
COPY weed /usr/bin/weed
RUN chmod +x /usr/bin/weed && \
chown seaweed:seaweed /usr/bin/weed && \
mkdir -p /etc/seaweedfs /data/filerldb2 && \
chown -R seaweed:seaweed /etc/seaweedfs /data && \
chmod 755 /data /etc/seaweedfs /data/filerldb2
WORKDIR /data
USER seaweed
-79
View File
@@ -1,79 +0,0 @@
# S3 Proxy Signature Verification Test
Integration test that verifies S3 signature verification works correctly when
SeaweedFS is deployed behind a reverse proxy (nginx).
## What it tests
- S3 operations (create bucket, put/get/head/list/delete) through an nginx
reverse proxy with `X-Forwarded-Host`, `X-Forwarded-Port`, and
`X-Forwarded-Proto` headers
- SeaweedFS configured with `-s3.externalUrl=http://localhost:9000` so the
signature verification uses the client-facing host instead of the internal
backend address
## Architecture
```text
AWS CLI (signs with Host: localhost:9000)
|
v
nginx (:9000)
| proxy_pass → seaweedfs:8333
| Sets: X-Forwarded-Host: localhost
| X-Forwarded-Port: 9000
| X-Forwarded-Proto: http
v
SeaweedFS S3 (:8333, -s3.externalUrl=http://localhost:9000)
| externalHost = "localhost:9000" (parsed at startup)
| extractHostHeader() returns "localhost:9000"
| Matches what AWS CLI signed with
v
Signature verification succeeds
```
**Note:** When `-s3.externalUrl` is configured, direct access to the backend
port (8333) will fail signature verification because the client signs with a
different Host header than what `externalUrl` specifies. This is expected —
all S3 traffic should go through the proxy.
## Prerequisites
- Docker and Docker Compose
- AWS CLI v2 (on host or via Docker, see below)
## Running
```bash
# Build the weed binary first (from repo root):
cd /path/to/seaweedfs
go build -o test/s3/proxy_signature/weed ./weed
cd test/s3/proxy_signature
# Start services
docker compose up -d --build
# Option A: Run test with aws CLI installed locally
./test.sh
# Option B: Run test without aws CLI (uses Docker container)
docker run --rm --network host --entrypoint "" amazon/aws-cli:latest \
bash < test.sh
# Tear down
docker compose down
# Clean up the weed binary
rm -f weed
```
## Troubleshooting
If signature verification fails through the proxy, check:
1. nginx is setting `X-Forwarded-Host` and `X-Forwarded-Port` correctly
2. SeaweedFS is started with `-s3.externalUrl` matching the client endpoint
3. The AWS CLI endpoint URL matches the proxy address
You can also set the `S3_EXTERNAL_URL` environment variable instead of the
`-s3.externalUrl` flag.
@@ -1,28 +0,0 @@
services:
seaweedfs:
build:
context: .
dockerfile: Dockerfile
command: >
/usr/bin/weed mini
-s3.config=/etc/seaweedfs/s3.json
-s3.externalUrl=http://localhost:9000
-ip=seaweedfs
volumes:
- ./s3.json:/etc/seaweedfs/s3.json:ro
healthcheck:
test: ["CMD", "curl", "-sf", "http://seaweedfs:8333/status"]
interval: 3s
timeout: 2s
retries: 20
start_period: 5s
nginx:
image: nginx:alpine
ports:
- "9000:9000"
volumes:
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
depends_on:
seaweedfs:
condition: service_healthy
-23
View File
@@ -1,23 +0,0 @@
server {
listen 9000;
server_name localhost;
# Allow large uploads
client_max_body_size 64m;
location / {
proxy_pass http://seaweedfs:8333;
# Standard reverse proxy headers — this is what Kong, Traefik, etc. do
proxy_set_header Host $host:$server_port;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
proxy_buffering off;
proxy_request_buffering off;
}
}
-20
View File
@@ -1,20 +0,0 @@
{
"identities": [
{
"name": "test_admin",
"credentials": [
{
"accessKey": "test_access_key",
"secretKey": "test_secret_key"
}
],
"actions": [
"Admin",
"Read",
"List",
"Tagging",
"Write"
]
}
]
}
-132
View File
@@ -1,132 +0,0 @@
#!/usr/bin/env bash
#
# Integration test for S3 signature verification behind a reverse proxy.
#
# Usage:
# # With aws CLI installed locally:
# docker compose up -d --build && ./test.sh && docker compose down
#
# # Without aws CLI (runs test inside a container):
# docker compose up -d --build
# docker run --rm --network host --entrypoint "" amazon/aws-cli:latest \
# bash < test.sh
# docker compose down
#
# This script tests S3 operations through an nginx reverse proxy to verify
# that signature verification works correctly when SeaweedFS is configured
# with -s3.externalUrl=http://localhost:9000.
#
set -euo pipefail
PROXY_ENDPOINT="http://localhost:9000"
ACCESS_KEY="test_access_key"
SECRET_KEY="test_secret_key"
REGION="us-east-1"
BUCKET="test-proxy-sig-$$"
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m'
pass() { echo -e "${GREEN}PASS${NC}: $1"; }
fail() { echo -e "${RED}FAIL${NC}: $1"; exit 1; }
# Helper: run aws s3api command against a given endpoint
s3() {
local endpoint="$1"
shift
aws s3api \
--endpoint-url "$endpoint" \
--region "$REGION" \
--no-verify-ssl \
"$@" 2>&1
}
export AWS_ACCESS_KEY_ID="$ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$SECRET_KEY"
echo "=== S3 Proxy Signature Verification Test ==="
echo ""
echo "Testing S3 access through nginx reverse proxy at $PROXY_ENDPOINT"
echo "SeaweedFS configured with -s3.externalUrl=http://localhost:9000"
echo "AWS CLI signs requests with Host: localhost:9000"
echo ""
# Wait for proxy to be ready
echo "Waiting for nginx proxy to be ready..."
for i in $(seq 1 30); do
# Use aws CLI for health check if curl is missing
if command -v curl >/dev/null 2>&1; then
http_code=$(curl -s -o /dev/null -w "%{http_code}" "$PROXY_ENDPOINT/" 2>/dev/null || echo "000")
case $http_code in
200|403|405) break ;;
esac
else
if aws s3api list-buckets --endpoint-url "$PROXY_ENDPOINT" --no-sign-request >/dev/null 2>&1; then
break
fi
fi
if [ "$i" -eq 30 ]; then
fail "Proxy did not become ready in time"
fi
echo "Waiting for proxy $i/30..."
sleep 1
done
echo "Proxy is ready."
echo ""
# --- Test 1: Bucket operations through proxy ---
echo "--- Test 1: Bucket operations through proxy ---"
s3 "$PROXY_ENDPOINT" create-bucket --bucket "$BUCKET" > /dev/null \
&& pass "create-bucket" \
|| fail "create-bucket — signature verification likely failed"
s3 "$PROXY_ENDPOINT" list-buckets > /dev/null \
&& pass "list-buckets" \
|| fail "list-buckets"
echo ""
# --- Test 2: Object CRUD through proxy ---
echo "--- Test 2: Object CRUD through proxy ---"
echo "hello-from-proxy" > /tmp/test-proxy-sig.txt
s3 "$PROXY_ENDPOINT" put-object --bucket "$BUCKET" --key "test.txt" --body /tmp/test-proxy-sig.txt > /dev/null \
&& pass "put-object" \
|| fail "put-object"
s3 "$PROXY_ENDPOINT" head-object --bucket "$BUCKET" --key "test.txt" > /dev/null \
&& pass "head-object" \
|| fail "head-object"
s3 "$PROXY_ENDPOINT" list-objects-v2 --bucket "$BUCKET" > /dev/null \
&& pass "list-objects-v2" \
|| fail "list-objects-v2"
s3 "$PROXY_ENDPOINT" get-object --bucket "$BUCKET" --key "test.txt" /tmp/test-proxy-sig-get.txt > /dev/null \
&& pass "get-object" \
|| fail "get-object"
# Verify content round-trip
CONTENT=$(cat /tmp/test-proxy-sig-get.txt)
if [ "$CONTENT" = "hello-from-proxy" ]; then
pass "content integrity (round-trip)"
else
fail "content mismatch: got \"$CONTENT\", expected \"hello-from-proxy\""
fi
echo ""
# --- Test 3: Delete operations through proxy ---
echo "--- Test 3: Delete through proxy ---"
s3 "$PROXY_ENDPOINT" delete-object --bucket "$BUCKET" --key "test.txt" > /dev/null \
&& pass "delete-object" \
|| fail "delete-object"
s3 "$PROXY_ENDPOINT" delete-bucket --bucket "$BUCKET" > /dev/null \
&& pass "delete-bucket" \
|| fail "delete-bucket"
echo ""
# Cleanup temp files
rm -f /tmp/test-proxy-sig.txt /tmp/test-proxy-sig-get.txt
echo "=== All tests passed ==="
+118 -466
View File
@@ -1,32 +1,15 @@
package lakekeeper
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
type TestEnvironment struct {
@@ -48,19 +31,15 @@ type TestEnvironment struct {
secretKey string
}
const (
lakekeeperRegion = "us-east-1"
lakekeeperRoleArn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
lakekeeperSessionName = "lakekeeper-session"
)
type lakekeeperSession struct {
endpoint string
region string
creds aws.Credentials
}
func TestLakekeeperIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !testutil.HasDocker() {
t.Skip("Docker not available, skipping Lakekeeper integration test")
}
env := NewTestEnvironment(t)
defer env.Cleanup(t)
@@ -72,18 +51,6 @@ func TestLakekeeperIntegration(t *testing.T) {
runLakekeeperRepro(t, env)
}
func TestLakekeeperTableBucketIntegration(t *testing.T) {
env := NewTestEnvironment(t)
defer env.Cleanup(t)
fmt.Printf(">>> Starting SeaweedFS with Lakekeeper configuration...\n")
env.StartSeaweedFS(t)
fmt.Printf(">>> SeaweedFS started.\n")
// Run python script in docker to test STS and S3 Tables operations
runLakekeeperTableBucketRepro(t, env)
}
func NewTestEnvironment(t *testing.T) *TestEnvironment {
t.Helper()
@@ -251,439 +218,124 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
func runLakekeeperRepro(t *testing.T, env *TestEnvironment) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
scriptContent := fmt.Sprintf(`
import boto3
import botocore.config
import botocore
from botocore.exceptions import ClientError
import os
import sys
import time
import logging
session, err := newLakekeeperSession(ctx, env)
if err != nil {
t.Fatalf("AssumeRole failed: %v", err)
# Enable botocore debug logging to see signature calculation
logging.basicConfig(level=logging.DEBUG)
botocore.session.get_session().set_debug_logger()
print("Starting Lakekeeper repro test...")
endpoint_url = "http://host.docker.internal:%d"
access_key = "%s"
secret_key = "%s"
region = "us-east-1"
print(f"Connecting to {endpoint_url}")
try:
config = botocore.config.Config(
retries={'max_attempts': 3}
)
sts = boto3.client(
'sts',
endpoint_url=endpoint_url,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region,
config=config
)
role_arn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
session_name = "lakekeeper-session"
print(f"Calling AssumeRole on {role_arn} with POST body...")
# Standard boto3 call sends parameters in POST body
response = sts.assume_role(
RoleArn=role_arn,
RoleSessionName=session_name
)
creds = response['Credentials']
access_key_id = creds['AccessKeyId']
secret_access_key = creds['SecretAccessKey']
session_token = creds['SessionToken']
print(f"Success! Got credentials with prefix: {access_key_id[:4]}")
if not access_key_id.startswith("ASIA"):
print(f"FAILED: Expected ASIA prefix, got {access_key_id}")
sys.exit(1)
print("Verifying S3 operations with vended credentials...")
s3 = boto3.client(
's3',
endpoint_url=endpoint_url,
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
aws_session_token=session_token,
region_name=region,
config=config
)
bucket = "lakekeeper-vended-bucket"
print(f"Creating bucket {bucket}...")
s3.create_bucket(Bucket=bucket)
print("Listing buckets...")
response = s3.list_buckets()
buckets = [b['Name'] for b in response['Buckets']]
print(f"Found buckets: {buckets}")
if bucket not in buckets:
print(f"FAILED: Bucket {bucket} not found in list")
sys.exit(1)
print("SUCCESS: Lakekeeper flow verified!")
sys.exit(0)
except Exception as e:
print(f"FAILED: {e}")
# Print more details if it is a ClientError
if hasattr(e, 'response'):
print(f"Response: {e.response}")
sys.exit(1)
`, env.s3Port, env.accessKey, env.secretKey)
scriptPath := filepath.Join(env.dataDir, "lakekeeper_repro.py")
if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
t.Fatalf("Failed to write python script: %v", err)
}
s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
if err != nil {
t.Fatalf("Create S3 client failed: %v", err)
}
containerName := "seaweed-lakekeeper-client-" + fmt.Sprintf("%d", time.Now().UnixNano())
bucketName := fmt.Sprintf("lakekeeper-vended-bucket-%d", time.Now().UnixNano())
if _, err := s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
}); err != nil {
t.Fatalf("CreateBucket failed: %v", err)
}
// Create a context with timeout for the docker run command
dockerCtx, dockerCancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer dockerCancel()
bucketCreated := true
defer func() {
if !bucketCreated {
return
}
_, _ = s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
Bucket: aws.String(bucketName),
})
}()
listResp, err := s3Client.ListBuckets(ctx, &s3.ListBucketsInput{})
if err != nil {
t.Fatalf("ListBuckets failed: %v", err)
}
found := false
for _, bucket := range listResp.Buckets {
if aws.ToString(bucket.Name) == bucketName {
found = true
break
}
}
if !found {
t.Fatalf("Bucket %s not found in list", bucketName)
}
if _, err := s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
Bucket: aws.String(bucketName),
}); err != nil {
t.Fatalf("DeleteBucket failed: %v", err)
}
bucketCreated = false
}
func runLakekeeperTableBucketRepro(t *testing.T, env *TestEnvironment) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
session, err := newLakekeeperSession(ctx, env)
if err != nil {
t.Fatalf("AssumeRole failed: %v", err)
}
client := newS3TablesClient(session.endpoint, session.region, session.creds)
bucketName := fmt.Sprintf("lakekeeper-table-bucket-%d", time.Now().UnixNano())
bucketARN, err := client.CreateTableBucket(ctx, bucketName)
if err != nil {
t.Fatalf("CreateTableBucket failed: %v", err)
}
bucketDeleted := false
namespaceCreated := false
tableCreated := false
namespaceName := fmt.Sprintf("lakekeeper_ns_%d", time.Now().UnixNano())
tableName := fmt.Sprintf("lakekeeper_table_%d", time.Now().UnixNano())
defer func() {
if tableCreated {
if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
t.Logf("Failed to delete table: %v", err)
}
}
if namespaceCreated {
if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
t.Logf("Failed to delete namespace: %v", err)
}
}
if bucketDeleted {
return
}
if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
t.Logf("Failed to delete table bucket: %v", err)
}
}()
buckets, err := client.ListTableBuckets(ctx)
if err != nil {
t.Fatalf("ListTableBuckets failed: %v", err)
}
found := false
for _, b := range buckets {
if b.Name == bucketName {
found = true
break
}
}
if !found {
t.Fatalf("Created table bucket %s not found in list", bucketName)
}
if _, err := client.GetTableBucket(ctx, bucketARN); err != nil {
t.Fatalf("GetTableBucket failed: %v", err)
}
if err := client.CreateNamespace(ctx, bucketARN, namespaceName); err != nil {
t.Fatalf("CreateNamespace failed: %v", err)
}
namespaceCreated = true
if err := client.CreateTable(ctx, bucketARN, namespaceName, tableName); err != nil {
t.Fatalf("CreateTable failed: %v", err)
}
tableCreated = true
s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
if err != nil {
t.Fatalf("Create S3 client failed: %v", err)
}
objectKey := fmt.Sprintf("%s/%s/data/part-%d.parquet", namespaceName, tableName, time.Now().UnixNano())
createResp, err := s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("CreateMultipartUpload failed: %v", err)
}
uploadID := aws.ToString(createResp.UploadId)
multipartCompleted := false
defer func() {
if uploadID == "" || multipartCompleted {
return
}
_, _ = s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
})
}()
partSize := 5 * 1024 * 1024
part1 := bytes.Repeat([]byte("a"), partSize)
part2 := bytes.Repeat([]byte("b"), 1024*1024)
part1Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(1),
Body: bytes.NewReader(part1),
})
if err != nil {
t.Fatalf("UploadPart 1 failed: %v", err)
}
part2Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(2),
Body: bytes.NewReader(part2),
})
if err != nil {
t.Fatalf("UploadPart 2 failed: %v", err)
}
_, err = s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
MultipartUpload: &s3types.CompletedMultipartUpload{
Parts: []s3types.CompletedPart{
{
ETag: part1Resp.ETag,
PartNumber: aws.Int32(1),
},
{
ETag: part2Resp.ETag,
PartNumber: aws.Int32(2),
},
},
},
})
if err != nil {
t.Fatalf("CompleteMultipartUpload failed: %v", err)
}
multipartCompleted = true
headResp, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("HeadObject after multipart upload failed: %v", err)
}
expectedSize := int64(len(part1) + len(part2))
if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
}
if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
t.Fatalf("DeleteTable failed: %v", err)
}
tableCreated = false
if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
t.Fatalf("DeleteNamespace failed: %v", err)
}
namespaceCreated = false
if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
t.Fatalf("DeleteTableBucket failed: %v", err)
}
bucketDeleted = true
if _, err := client.GetTableBucket(ctx, bucketARN); err == nil {
t.Fatalf("expected GetTableBucket to fail after deletion")
}
}
type s3TablesClient struct {
endpoint string
region string
creds aws.Credentials
httpClient *http.Client
}
func newS3TablesClient(endpoint, region string, creds aws.Credentials) *s3TablesClient {
return &s3TablesClient{
endpoint: endpoint,
region: region,
creds: creds,
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
func (c *s3TablesClient) CreateTableBucket(ctx context.Context, name string) (string, error) {
req := &s3tables.CreateTableBucketRequest{Name: name}
var resp s3tables.CreateTableBucketResponse
if err := c.doRequest(ctx, "CreateTableBucket", http.MethodPut, "/buckets", req, &resp); err != nil {
return "", err
}
return resp.ARN, nil
}
func (c *s3TablesClient) GetTableBucket(ctx context.Context, arn string) (*s3tables.GetTableBucketResponse, error) {
path := "/buckets/" + url.PathEscape(arn)
var resp s3tables.GetTableBucketResponse
if err := c.doRequest(ctx, "GetTableBucket", http.MethodGet, path, nil, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func (c *s3TablesClient) ListTableBuckets(ctx context.Context) ([]s3tables.TableBucketSummary, error) {
var resp s3tables.ListTableBucketsResponse
if err := c.doRequest(ctx, "ListTableBuckets", http.MethodGet, "/buckets", nil, &resp); err != nil {
return nil, err
}
return resp.TableBuckets, nil
}
func (c *s3TablesClient) DeleteTableBucket(ctx context.Context, arn string) error {
path := "/buckets/" + url.PathEscape(arn)
return c.doRequest(ctx, "DeleteTableBucket", http.MethodDelete, path, nil, nil)
}
func (c *s3TablesClient) CreateNamespace(ctx context.Context, bucketARN, namespace string) error {
req := &s3tables.CreateNamespaceRequest{
Namespace: []string{namespace},
}
path := "/namespaces/" + url.PathEscape(bucketARN)
return c.doRequest(ctx, "CreateNamespace", http.MethodPut, path, req, nil)
}
func (c *s3TablesClient) DeleteNamespace(ctx context.Context, bucketARN, namespace string) error {
path := "/namespaces/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
return c.doRequest(ctx, "DeleteNamespace", http.MethodDelete, path, nil, nil)
}
func (c *s3TablesClient) CreateTable(ctx context.Context, bucketARN, namespace, name string) error {
req := &s3tables.CreateTableRequest{
Name: name,
Format: "ICEBERG",
}
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
return c.doRequest(ctx, "CreateTable", http.MethodPut, path, req, nil)
}
func (c *s3TablesClient) DeleteTable(ctx context.Context, bucketARN, namespace, name string) error {
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace) + "/" + url.PathEscape(name)
return c.doRequest(ctx, "DeleteTable", http.MethodDelete, path, nil, nil)
}
func (c *s3TablesClient) doRequest(ctx context.Context, operation, method, path string, body interface{}, out interface{}) error {
var bodyBytes []byte
if body != nil {
encoded, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("%s: marshal request: %w", operation, err)
}
bodyBytes = encoded
}
req, err := http.NewRequestWithContext(ctx, method, c.endpoint+path, bytes.NewReader(bodyBytes))
if err != nil {
return fmt.Errorf("%s: create request: %w", operation, err)
}
if body != nil {
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
}
req.Host = req.URL.Host
req.Header.Set("Host", req.URL.Host)
payloadHash := sha256.Sum256(bodyBytes)
if err := v4.NewSigner().SignHTTP(ctx, c.creds, req, hex.EncodeToString(payloadHash[:]), "s3tables", c.region, time.Now()); err != nil {
return fmt.Errorf("%s: sign request: %w", operation, err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("%s: request failed: %w", operation, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
bodyBytes, readErr := io.ReadAll(resp.Body)
if readErr != nil {
return fmt.Errorf("%s failed with status %d and could not read response body: %v", operation, resp.StatusCode, readErr)
}
var errResp s3tables.S3TablesError
if jsonErr := json.Unmarshal(bodyBytes, &errResp); jsonErr == nil && (errResp.Type != "" || errResp.Message != "") {
return fmt.Errorf("%s failed: %s - %s", operation, errResp.Type, errResp.Message)
}
return fmt.Errorf("%s failed with status %d: %s", operation, resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
}
if out == nil {
return nil
}
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
return fmt.Errorf("%s: decode response: %w", operation, err)
}
return nil
}
func assumeRole(ctx context.Context, endpoint, region, accessKey, secretKey, roleArn, sessionName string) (aws.Credentials, error) {
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == sts.ServiceID {
return aws.Endpoint{
URL: endpoint,
SigningRegion: region,
HostnameImmutable: true,
}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
config.WithEndpointResolverWithOptions(resolver),
cmd := exec.CommandContext(dockerCtx, "docker", "run", "--rm",
"--name", containerName,
"--add-host", "host.docker.internal:host-gateway",
"-v", fmt.Sprintf("%s:/work", env.dataDir),
"python:3",
"/bin/bash", "-c", "pip install boto3 && python /work/lakekeeper_repro.py",
)
if err != nil {
return aws.Credentials{}, err
}
client := sts.NewFromConfig(cfg)
resp, err := client.AssumeRole(ctx, &sts.AssumeRoleInput{
RoleArn: aws.String(roleArn),
RoleSessionName: aws.String(sessionName),
})
output, err := cmd.CombinedOutput()
if err != nil {
return aws.Credentials{}, err
}
if resp.Credentials == nil {
return aws.Credentials{}, fmt.Errorf("missing credentials in AssumeRole response")
}
return aws.Credentials{
AccessKeyID: aws.ToString(resp.Credentials.AccessKeyId),
SecretAccessKey: aws.ToString(resp.Credentials.SecretAccessKey),
SessionToken: aws.ToString(resp.Credentials.SessionToken),
Source: "lakekeeper-sts",
}, nil
}
func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials) (*s3.Client, error) {
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID {
return aws.Endpoint{
URL: endpoint,
SigningRegion: region,
HostnameImmutable: true,
}, nil
if dockerCtx.Err() == context.DeadlineExceeded {
t.Fatalf("Lakekeeper repro client timed out after 5 minutes\nOutput:\n%s", string(output))
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
config.WithEndpointResolverWithOptions(resolver),
)
if err != nil {
return nil, err
t.Fatalf("Lakekeeper repro client failed: %v\nOutput:\n%s", err, string(output))
}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = true
}), nil
}
func newLakekeeperSession(ctx context.Context, env *TestEnvironment) (lakekeeperSession, error) {
endpoint := fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
creds, err := assumeRole(ctx, endpoint, lakekeeperRegion, env.accessKey, env.secretKey, lakekeeperRoleArn, lakekeeperSessionName)
if err != nil {
return lakekeeperSession{}, err
}
return lakekeeperSession{
endpoint: endpoint,
region: lakekeeperRegion,
creds: creds,
}, nil
t.Logf("Lakekeeper repro client output:\n%s", string(output))
}
-377
View File
@@ -1,377 +0,0 @@
package polaris
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
)
const (
polarisImage = "apache/polaris:latest"
polarisRealm = "POLARIS"
polarisRootClientID = "root"
polarisRootClientSecret = "s3cr3t"
polarisRegion = "us-east-1"
polarisRoleArn = "arn:aws:iam::000000000000:role/PolarisVendedRole"
polarisSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test signing key
)
type TestEnvironment struct {
seaweedDir string
weedBinary string
dataDir string
bindIP string
s3Port int
s3GrpcPort int
masterPort int
masterGrpcPort int
filerPort int
filerGrpcPort int
volumePort int
volumeGrpcPort int
polarisPort int
polarisAdminPort int
weedProcess *exec.Cmd
weedCancel context.CancelFunc
polarisContainer string
accessKey string
secretKey string
}
func NewTestEnvironment(t *testing.T) *TestEnvironment {
t.Helper()
if !testutil.HasDocker() {
t.Skip("Docker is required for Polaris integration tests")
}
wd, err := os.Getwd()
if err != nil {
t.Fatalf("Failed to get working directory: %v", err)
}
seaweedDir := wd
for i := 0; i < 6; i++ {
if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
break
}
seaweedDir = filepath.Dir(seaweedDir)
}
weedBinary := filepath.Join(seaweedDir, "weed", "weed")
if _, err := os.Stat(weedBinary); err != nil {
weedBinary = "weed"
if _, err := exec.LookPath(weedBinary); err != nil {
t.Skip("weed binary not found, skipping integration test")
}
}
dataDir, err := os.MkdirTemp("", "seaweed-polaris-test-*")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
bindIP := testutil.FindBindIP()
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
return &TestEnvironment{
seaweedDir: seaweedDir,
weedBinary: weedBinary,
dataDir: dataDir,
bindIP: bindIP,
s3Port: s3Port,
s3GrpcPort: s3GrpcPort,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
filerPort: filerPort,
filerGrpcPort: filerGrpcPort,
volumePort: volumePort,
volumeGrpcPort: volumeGrpcPort,
polarisPort: polarisPort,
polarisAdminPort: polarisAdminPort,
accessKey: "admin",
secretKey: "admin",
}
}
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Helper()
iamConfigPath := filepath.Join(env.dataDir, "iam.json")
iamConfig := fmt.Sprintf(`{
"identities": [
{
"name": "admin",
"credentials": [
{
"accessKey": "%s",
"secretKey": "%s"
}
],
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
}
],
"sts": {
"tokenDuration": "12h",
"maxSessionLength": "24h",
"issuer": "seaweedfs-sts",
"signingKey": "%s"
},
"roles": [
{
"roleName": "PolarisVendedRole",
"roleArn": "%s",
"trustPolicy": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": "*",
"Action": "sts:AssumeRole"
}
]
},
"attachedPolicies": ["FullAccess"]
}
],
"policies": [
{
"name": "FullAccess",
"document": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "*",
"Resource": "*"
}
]
}
}
]
}`, env.accessKey, env.secretKey, polarisSigningKey, polarisRoleArn)
if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
t.Fatalf("Failed to create IAM config: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
env.weedCancel = cancel
cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini",
"-master.port", fmt.Sprintf("%d", env.masterPort),
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
"-volume.port", fmt.Sprintf("%d", env.volumePort),
"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
"-filer.port", fmt.Sprintf("%d", env.filerPort),
"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
"-s3.port", fmt.Sprintf("%d", env.s3Port),
"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
"-s3.config", iamConfigPath,
"-s3.iam.config", iamConfigPath,
"-s3.iam.readOnly=false",
"-ip", env.bindIP,
"-ip.bind", "0.0.0.0",
"-dir", env.dataDir,
)
cmd.Dir = env.dataDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
t.Fatalf("Failed to start SeaweedFS: %v", err)
}
env.weedProcess = cmd
if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
t.Fatalf("S3 API failed to become ready")
}
}
func (env *TestEnvironment) StartPolaris(t *testing.T) {
t.Helper()
containerName := fmt.Sprintf("seaweed-polaris-%d", time.Now().UnixNano())
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
cmd := exec.CommandContext(ctx, "docker", "run", "-d", "--rm",
"--name", containerName,
"--add-host", "host.docker.internal:host-gateway",
"-p", fmt.Sprintf("%d:8181", env.polarisPort),
"-p", fmt.Sprintf("%d:8182", env.polarisAdminPort),
"-e", fmt.Sprintf("AWS_REGION=%s", polarisRegion),
"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", env.accessKey),
"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", env.secretKey),
"-e", fmt.Sprintf("POLARIS_BOOTSTRAP_CREDENTIALS=%s,%s,%s", polarisRealm, polarisRootClientID, polarisRootClientSecret),
"-e", fmt.Sprintf("polaris.realm-context.realms=%s", polarisRealm),
"-e", "quarkus.otel.sdk.disabled=true",
polarisImage,
)
output, err := cmd.CombinedOutput()
if err != nil {
if ctx.Err() == context.DeadlineExceeded {
t.Fatalf("Timed out waiting for Polaris container: %v\nOutput:\n%s", ctx.Err(), string(output))
}
t.Fatalf("Failed to start Polaris: %v\nOutput:\n%s", err, string(output))
}
env.polarisContainer = containerName
if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/q/health", env.polarisAdminPort), 60*time.Second) {
logCtx, logCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer logCancel()
logs, _ := exec.CommandContext(logCtx, "docker", "logs", env.polarisContainer).CombinedOutput()
t.Fatalf("Polaris failed to become ready\nLogs:\n%s", string(logs))
}
}
func (env *TestEnvironment) Cleanup(t *testing.T) {
t.Helper()
if env.weedCancel != nil {
env.weedCancel()
}
if env.weedProcess != nil {
time.Sleep(1 * time.Second)
_ = env.weedProcess.Wait()
}
if env.polarisContainer != "" {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_ = exec.CommandContext(ctx, "docker", "rm", "-f", env.polarisContainer).Run()
}
if env.dataDir != "" {
_ = os.RemoveAll(env.dataDir)
}
}
func (env *TestEnvironment) polarisEndpoint() string {
return fmt.Sprintf("http://127.0.0.1:%d", env.polarisPort)
}
func (env *TestEnvironment) s3Endpoint() string {
return fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
}
func (env *TestEnvironment) s3InternalEndpoint() string {
return fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
}
type polarisHTTPClient struct {
baseURL string
realm string
token string
httpClient *http.Client
}
func newPolarisHTTPClient(baseURL, realm, token string) *polarisHTTPClient {
return &polarisHTTPClient{
baseURL: baseURL,
realm: realm,
token: token,
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
func (c *polarisHTTPClient) doJSON(ctx context.Context, method, path string, body interface{}, out interface{}) error {
return c.doJSONWithHeaders(ctx, method, path, body, out, nil)
}
func (c *polarisHTTPClient) doJSONWithHeaders(ctx context.Context, method, path string, body interface{}, out interface{}, headers map[string]string) error {
var reader io.Reader
if body != nil {
encoded, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("encode request body: %w", err)
}
reader = bytes.NewReader(encoded)
}
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)
if err != nil {
return fmt.Errorf("create request: %w", err)
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
req.Header.Set("Accept", "application/json")
if c.realm != "" {
req.Header.Set("Polaris-Realm", c.realm)
}
if c.token != "" {
req.Header.Set("Authorization", "Bearer "+c.token)
}
for key, value := range headers {
req.Header.Set(key, value)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
bodyBytes, readErr := io.ReadAll(resp.Body)
if readErr != nil {
return fmt.Errorf("request failed with status %d and could not read response body: %w", resp.StatusCode, readErr)
}
return fmt.Errorf("request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
}
if out == nil {
return nil
}
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
return fmt.Errorf("decode response: %w", err)
}
return nil
}
func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials, pathStyle bool) (*s3.Client, error) {
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID {
return aws.Endpoint{
URL: endpoint,
SigningRegion: region,
HostnameImmutable: true,
}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
config.WithEndpointResolverWithOptions(resolver),
)
if err != nil {
return nil, err
}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = pathStyle
}), nil
}
-764
View File
@@ -1,764 +0,0 @@
package polaris
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type polarisSession struct {
catalogName string
bucketName string
token string
baseLocation string
}
type polarisTableSetup struct {
namespace string
table string
dataKeyPrefix string
s3Client *s3.Client
}
type polarisCatalogClient struct {
http *polarisHTTPClient
catalog string
}
type polarisCredentials struct {
ClientID string `json:"clientId"`
ClientSecret string `json:"clientSecret"`
}
type createPrincipalResponse struct {
Credentials polarisCredentials `json:"credentials"`
}
type createCatalogRequest struct {
Catalog polarisCatalog `json:"catalog"`
}
type polarisCatalog struct {
Name string `json:"name"`
Type string `json:"type"`
ReadOnly bool `json:"readOnly"`
Properties map[string]string `json:"properties"`
StorageConfigInfo polarisStorageConfig `json:"storageConfigInfo"`
}
type polarisStorageConfig struct {
StorageType string `json:"storageType"`
AllowedLocations []string `json:"allowedLocations"`
Endpoint string `json:"endpoint"`
EndpointInternal string `json:"endpointInternal"`
StsEndpoint string `json:"stsEndpoint"`
PathStyleAccess bool `json:"pathStyleAccess"`
RoleArn string `json:"roleArn"`
Region string `json:"region"`
}
type createNamespaceRequest struct {
Namespace []string `json:"namespace"`
}
type createTableRequest struct {
Name string `json:"name"`
Location string `json:"location"`
Schema icebergSchema `json:"schema"`
PartitionSpec icebergPartition `json:"partition-spec"`
SortOrder icebergSortOrder `json:"sort-order"`
Properties map[string]string `json:"properties"`
}
type icebergSchema struct {
Type string `json:"type"`
SchemaID int `json:"schema-id"`
Fields []icebergSchemaField `json:"fields"`
}
type icebergSchemaField struct {
ID int `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Required bool `json:"required"`
}
type icebergPartition struct {
SpecID int `json:"spec-id"`
Fields []icebergPartitionField `json:"fields"`
}
type icebergPartitionField struct {
SourceID int `json:"source-id"`
FieldID int `json:"field-id"`
Name string `json:"name"`
Transform string `json:"transform"`
}
type icebergSortOrder struct {
OrderID int `json:"order-id"`
Fields []icebergSortField `json:"fields"`
}
type icebergSortField struct {
SourceID int `json:"source-id"`
Direction string `json:"direction"`
NullOrder string `json:"null-order"`
}
type loadTableResponse struct {
Config map[string]string `json:"config"`
StorageCredentials []storageCredential `json:"storage-credentials"`
}
type loadCredentialsResponse struct {
StorageCredentials []storageCredential `json:"storage-credentials"`
}
type storageCredential struct {
Prefix string `json:"prefix"`
Config map[string]string `json:"config"`
}
func bootstrapPolarisTest(t *testing.T, env *TestEnvironment) (context.Context, context.CancelFunc, polarisSession, *polarisTableSetup, func()) {
t.Helper()
t.Logf(">>> Starting SeaweedFS with Polaris configuration...")
env.StartSeaweedFS(t)
t.Logf(">>> SeaweedFS started.")
t.Logf(">>> Starting Polaris...")
env.StartPolaris(t)
t.Logf(">>> Polaris started.")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
session := newPolarisSession(t, ctx, env)
setup, cleanup := setupPolarisTable(t, ctx, env, session)
return ctx, cancel, session, setup, cleanup
}
func TestPolarisIntegration(t *testing.T) {
env := NewTestEnvironment(t)
defer env.Cleanup(t)
ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
defer cancel()
defer cleanup()
objectKey := fmt.Sprintf("%s/hello-%d.txt", setup.dataKeyPrefix, time.Now().UnixNano())
payload := []byte("polaris")
if _, err := setup.s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader(payload),
}); err != nil {
t.Fatalf("PutObject failed: %v", err)
}
listObjects, err := setup.s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(session.bucketName),
Prefix: aws.String(objectKey),
})
if err != nil {
t.Fatalf("ListObjectsV2 failed: %v", err)
}
found := false
for _, obj := range listObjects.Contents {
if aws.ToString(obj.Key) == objectKey {
found = true
break
}
}
if !found {
t.Fatalf("Object %s not found in list", objectKey)
}
getResp, err := setup.s3Client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("GetObject failed: %v", err)
}
body, err := io.ReadAll(getResp.Body)
_ = getResp.Body.Close()
if err != nil {
t.Fatalf("Read object body failed: %v", err)
}
if !bytes.Equal(body, payload) {
t.Fatalf("Unexpected object payload: got %q want %q", string(body), string(payload))
}
if _, err := setup.s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
}); err != nil {
t.Fatalf("DeleteObject failed: %v", err)
}
}
func TestPolarisTableIntegration(t *testing.T) {
env := NewTestEnvironment(t)
defer env.Cleanup(t)
ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
defer cancel()
defer cleanup()
objectKey := fmt.Sprintf("%s/part-%d.parquet", setup.dataKeyPrefix, time.Now().UnixNano())
createResp, err := setup.s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("CreateMultipartUpload failed: %v", err)
}
uploadID := aws.ToString(createResp.UploadId)
multipartCompleted := false
defer func() {
if uploadID == "" || multipartCompleted {
return
}
_, _ = setup.s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
})
}()
partSize := 5 * 1024 * 1024
part1 := bytes.Repeat([]byte("a"), partSize)
part2 := bytes.Repeat([]byte("b"), 1024*1024)
part1Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(1),
Body: bytes.NewReader(part1),
})
if err != nil {
t.Fatalf("UploadPart 1 failed: %v", err)
}
part2Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(2),
Body: bytes.NewReader(part2),
})
if err != nil {
t.Fatalf("UploadPart 2 failed: %v", err)
}
_, err = setup.s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
MultipartUpload: &s3types.CompletedMultipartUpload{
Parts: []s3types.CompletedPart{
{
ETag: part1Resp.ETag,
PartNumber: aws.Int32(1),
},
{
ETag: part2Resp.ETag,
PartNumber: aws.Int32(2),
},
},
},
})
if err != nil {
t.Fatalf("CompleteMultipartUpload failed: %v", err)
}
multipartCompleted = true
headResp, err := setup.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("HeadObject after multipart upload failed: %v", err)
}
expectedSize := int64(len(part1) + len(part2))
if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
}
}
func newPolarisSession(t *testing.T, ctx context.Context, env *TestEnvironment) polarisSession {
t.Helper()
adminCreds := aws.Credentials{
AccessKeyID: env.accessKey,
SecretAccessKey: env.secretKey,
Source: "polaris-admin",
}
adminS3, err := newS3Client(ctx, env.s3Endpoint(), polarisRegion, adminCreds, true)
if err != nil {
t.Fatalf("Create admin S3 client failed: %v", err)
}
bucketName := fmt.Sprintf("polaris-bucket-%d", time.Now().UnixNano())
if _, err := adminS3.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
}); err != nil {
t.Fatalf("CreateBucket failed: %v", err)
}
policyDoc := map[string]interface{}{
"Version": "2012-10-17",
"Statement": []map[string]interface{}{
{
"Sid": "AllowPolarisVendedAccess",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:*",
"Resource": []string{
fmt.Sprintf("arn:aws:s3:::%s", bucketName),
fmt.Sprintf("arn:aws:s3:::%s/polaris/*", bucketName),
},
},
},
}
policyBytes, err := json.Marshal(policyDoc)
if err != nil {
t.Fatalf("Failed to marshal bucket policy: %v", err)
}
if _, err := adminS3.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
Bucket: aws.String(bucketName),
Policy: aws.String(string(policyBytes)),
}); err != nil {
t.Fatalf("PutBucketPolicy failed: %v", err)
}
rootToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), polarisRootClientID, polarisRootClientSecret)
if err != nil {
t.Fatalf("Polaris root token request failed: %v", err)
}
managementClient := newPolarisHTTPClient(env.polarisEndpoint(), polarisRealm, rootToken)
catalogName := fmt.Sprintf("polaris_catalog_%d", time.Now().UnixNano())
baseLocation := fmt.Sprintf("s3://%s/polaris", bucketName)
catalogRequest := createCatalogRequest{
Catalog: polarisCatalog{
Name: catalogName,
Type: "INTERNAL",
ReadOnly: false,
Properties: map[string]string{
"default-base-location": baseLocation,
},
StorageConfigInfo: polarisStorageConfig{
StorageType: "S3",
AllowedLocations: []string{baseLocation},
Endpoint: env.s3Endpoint(),
EndpointInternal: env.s3InternalEndpoint(),
StsEndpoint: env.s3InternalEndpoint(),
PathStyleAccess: true,
RoleArn: polarisRoleArn,
Region: polarisRegion,
},
},
}
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/catalogs", catalogRequest, nil); err != nil {
t.Fatalf("Create catalog failed: %v", err)
}
principalName := fmt.Sprintf("polaris_user_%d", time.Now().UnixNano())
principalRoleName := fmt.Sprintf("polaris_principal_role_%d", time.Now().UnixNano())
catalogRoleName := fmt.Sprintf("polaris_catalog_role_%d", time.Now().UnixNano())
var principalResp createPrincipalResponse
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principals", map[string]interface{}{
"principal": map[string]interface{}{
"name": principalName,
"properties": map[string]string{},
},
}, &principalResp); err != nil {
t.Fatalf("Create principal failed: %v", err)
}
if principalResp.Credentials.ClientID == "" || principalResp.Credentials.ClientSecret == "" {
t.Fatalf("Missing principal credentials in response")
}
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principal-roles", map[string]interface{}{
"principalRole": map[string]interface{}{
"name": principalRoleName,
"properties": map[string]string{},
},
}, nil); err != nil {
t.Fatalf("Create principal role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPost, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles", url.PathEscape(catalogName)), map[string]interface{}{
"catalogRole": map[string]interface{}{
"name": catalogRoleName,
"properties": map[string]string{},
},
}, nil); err != nil {
t.Fatalf("Create catalog role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principals/%s/principal-roles", url.PathEscape(principalName)), map[string]interface{}{
"principalRole": map[string]interface{}{
"name": principalRoleName,
},
}, nil); err != nil {
t.Fatalf("Assign principal role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principal-roles/%s/catalog-roles/%s", url.PathEscape(principalRoleName), url.PathEscape(catalogName)), map[string]interface{}{
"catalogRole": map[string]interface{}{
"name": catalogRoleName,
},
}, nil); err != nil {
t.Fatalf("Assign catalog role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles/%s/grants", url.PathEscape(catalogName), url.PathEscape(catalogRoleName)), map[string]interface{}{
"type": "catalog",
"privilege": "CATALOG_MANAGE_CONTENT",
}, nil); err != nil {
t.Fatalf("Grant catalog privilege failed: %v", err)
}
userToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), principalResp.Credentials.ClientID, principalResp.Credentials.ClientSecret)
if err != nil {
t.Fatalf("Polaris user token request failed: %v", err)
}
return polarisSession{
catalogName: catalogName,
bucketName: bucketName,
token: userToken,
baseLocation: baseLocation,
}
}
func setupPolarisTable(t *testing.T, ctx context.Context, env *TestEnvironment, session polarisSession) (*polarisTableSetup, func()) {
t.Helper()
catalogClient := newPolarisCatalogClient(env.polarisEndpoint(), polarisRealm, session.token, session.catalogName)
namespace := fmt.Sprintf("polaris_ns_%d", time.Now().UnixNano())
table := fmt.Sprintf("polaris_table_%d", time.Now().UnixNano())
if err := catalogClient.CreateNamespace(ctx, namespace); err != nil {
t.Fatalf("CreateNamespace failed: %v", err)
}
location := fmt.Sprintf("%s/%s/%s", session.baseLocation, namespace, table)
if err := catalogClient.CreateTable(ctx, namespace, table, location); err != nil {
t.Fatalf("CreateTable failed: %v", err)
}
loadResp, err := catalogClient.LoadTable(ctx, namespace, table)
if err != nil {
t.Fatalf("LoadTable failed: %v", err)
}
credsResp, err := catalogClient.LoadCredentials(ctx, namespace, table)
if err != nil {
t.Fatalf("LoadCredentials failed: %v", err)
}
if len(credsResp.StorageCredentials) == 0 {
t.Fatalf("LoadCredentials returned no storage credentials")
}
credentialSource := &loadTableResponse{
Config: loadResp.Config,
StorageCredentials: credsResp.StorageCredentials,
}
dataPrefix := location + "/data"
creds, endpoint, region, pathStyle, err := extractS3Credentials(credentialSource, dataPrefix, env.s3Endpoint(), polarisRegion)
if err != nil {
t.Fatalf("Extract vended credentials failed: %v", err)
}
dataKeyPrefix, err := s3URIToKeyPrefix(dataPrefix, session.bucketName)
if err != nil {
t.Fatalf("Invalid data prefix %s: %v", dataPrefix, err)
}
s3Client, err := newS3Client(ctx, endpoint, region, creds, pathStyle)
if err != nil {
t.Fatalf("Create vended S3 client failed: %v", err)
}
cleanup := func() {
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cleanupCancel()
if err := catalogClient.DeleteTable(cleanupCtx, namespace, table); err != nil {
t.Logf("DeleteTable failed: %v", err)
}
if err := catalogClient.DeleteNamespace(cleanupCtx, namespace); err != nil {
t.Logf("DeleteNamespace failed: %v", err)
}
}
return &polarisTableSetup{
namespace: namespace,
table: table,
dataKeyPrefix: dataKeyPrefix,
s3Client: s3Client,
}, cleanup
}
func fetchPolarisToken(ctx context.Context, baseURL, clientID, clientSecret string) (string, error) {
form := url.Values{}
form.Set("grant_type", "client_credentials")
form.Set("client_id", clientID)
form.Set("client_secret", clientSecret)
form.Set("scope", "PRINCIPAL_ROLE:ALL")
req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/api/catalog/v1/oauth/tokens", strings.NewReader(form.Encode()))
if err != nil {
return "", fmt.Errorf("create token request: %w", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("token request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, readErr := io.ReadAll(resp.Body)
if readErr != nil {
return "", fmt.Errorf("token request failed with status %d and reading body: %w", resp.StatusCode, readErr)
}
return "", fmt.Errorf("token request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
var tokenResp struct {
AccessToken string `json:"access_token"`
}
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
return "", fmt.Errorf("decode token response: %w", err)
}
if tokenResp.AccessToken == "" {
return "", fmt.Errorf("missing access token in response")
}
return tokenResp.AccessToken, nil
}
func newPolarisCatalogClient(baseURL, realm, token, catalog string) *polarisCatalogClient {
return &polarisCatalogClient{
http: newPolarisHTTPClient(baseURL, realm, token),
catalog: catalog,
}
}
func (c *polarisCatalogClient) CreateNamespace(ctx context.Context, namespace string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces", url.PathEscape(c.catalog))
req := createNamespaceRequest{Namespace: []string{namespace}}
return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
}
func (c *polarisCatalogClient) DeleteNamespace(ctx context.Context, namespace string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s", url.PathEscape(c.catalog), url.PathEscape(namespace))
return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
}
func (c *polarisCatalogClient) CreateTable(ctx context.Context, namespace, table, location string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables", url.PathEscape(c.catalog), url.PathEscape(namespace))
req := createTableRequest{
Name: table,
Location: location,
Schema: icebergSchema{
Type: "struct",
SchemaID: 0,
Fields: []icebergSchemaField{
{
ID: 1,
Name: "id",
Type: "long",
Required: false,
},
},
},
PartitionSpec: icebergPartition{
SpecID: 0,
Fields: []icebergPartitionField{},
},
SortOrder: icebergSortOrder{
OrderID: 0,
Fields: []icebergSortField{},
},
Properties: map[string]string{
"format-version": "2",
},
}
return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
}
func (c *polarisCatalogClient) DeleteTable(ctx context.Context, namespace, table string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
}
func (c *polarisCatalogClient) LoadTable(ctx context.Context, namespace, table string) (*loadTableResponse, error) {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
var resp loadTableResponse
headers := map[string]string{
"X-Iceberg-Access-Delegation": "vended-credentials",
}
if err := c.http.doJSONWithHeaders(ctx, http.MethodGet, path, nil, &resp, headers); err != nil {
return nil, err
}
return &resp, nil
}
func (c *polarisCatalogClient) LoadCredentials(ctx context.Context, namespace, table string) (*loadCredentialsResponse, error) {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s/credentials", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
var resp loadCredentialsResponse
if err := c.http.doJSON(ctx, http.MethodGet, path, nil, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoint, fallbackRegion string) (aws.Credentials, string, string, bool, error) {
credentialConfig, err := selectStorageConfig(load, targetPrefix)
if err != nil {
return aws.Credentials{}, "", "", false, err
}
lookupConfig := func(key string) string {
if load != nil && load.Config != nil {
if val, ok := load.Config[key]; ok && strings.TrimSpace(val) != "" {
return strings.TrimSpace(val)
}
}
if val, ok := credentialConfig[key]; ok && strings.TrimSpace(val) != "" {
return strings.TrimSpace(val)
}
return ""
}
accessKey := strings.TrimSpace(credentialConfig["s3.access-key-id"])
secretKey := strings.TrimSpace(credentialConfig["s3.secret-access-key"])
sessionToken := strings.TrimSpace(credentialConfig["s3.session-token"])
if accessKey == "" || secretKey == "" {
return aws.Credentials{}, "", "", false, fmt.Errorf("missing s3.access-key-id or s3.secret-access-key in selected storage credential")
}
endpoint := lookupConfig("s3.endpoint")
if endpoint == "" {
endpoint = fallbackEndpoint
}
if endpoint != "" && !strings.HasPrefix(endpoint, "http") {
endpoint = "http://" + endpoint
}
region := lookupConfig("client.region")
if region == "" {
region = fallbackRegion
}
pathStyle := true
if value := lookupConfig("s3.path-style-access"); value != "" {
pathStyle = strings.EqualFold(value, "true")
}
return aws.Credentials{
AccessKeyID: accessKey,
SecretAccessKey: secretKey,
SessionToken: sessionToken,
Source: "polaris-vended",
}, endpoint, region, pathStyle, nil
}
func selectStorageConfig(load *loadTableResponse, targetPrefix string) (map[string]string, error) {
if load == nil {
return nil, fmt.Errorf("load table response is nil")
}
switch len(load.StorageCredentials) {
case 0:
if load.Config == nil {
return nil, fmt.Errorf("polaris returned no storage credentials or config")
}
return load.Config, nil
case 1:
cred := load.StorageCredentials[0]
if cred.Config == nil {
return nil, fmt.Errorf("storage credential for prefix %s returned nil config", cred.Prefix)
}
return cred.Config, nil
default:
if targetPrefix == "" {
return nil, fmt.Errorf("multiple storage credentials (%d) returned but no target prefix provided", len(load.StorageCredentials))
}
normalizedTarget := normalizePrefix(targetPrefix)
if normalizedTarget == "" {
return nil, fmt.Errorf("target prefix %q normalized to empty string", targetPrefix)
}
var bestConfig map[string]string
bestLen := -1
for _, cred := range load.StorageCredentials {
if cred.Config == nil {
continue
}
prefix := normalizePrefix(cred.Prefix)
if prefix == "" {
if bestLen < 0 {
bestLen = 0
bestConfig = cred.Config
}
continue
}
if normalizedTarget == prefix || strings.HasPrefix(normalizedTarget, prefix+"/") {
if len(prefix) > bestLen {
bestLen = len(prefix)
bestConfig = cred.Config
}
}
}
if bestConfig != nil {
return bestConfig, nil
}
return nil, fmt.Errorf("none of the %d storage credentials matched prefix %s", len(load.StorageCredentials), targetPrefix)
}
}
func normalizePrefix(prefix string) string {
p := strings.TrimSpace(prefix)
p = strings.TrimSuffix(p, "/")
return p
}
func s3URIToKeyPrefix(uri, bucket string) (string, error) {
prefix := "s3://" + bucket + "/"
if !strings.HasPrefix(uri, prefix) {
return "", fmt.Errorf("uri %q does not match bucket %q", uri, bucket)
}
keyPrefix := strings.TrimPrefix(uri, prefix)
keyPrefix = strings.TrimPrefix(keyPrefix, "/")
if keyPrefix == "" {
return "", fmt.Errorf("empty key prefix in uri %q", uri)
}
return keyPrefix, nil
}
+7 -4
View File
@@ -23,6 +23,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
if err != nil {
t.Fatalf("create status request: %v", err)
}
statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1")
statusResp := framework.DoRequest(t, client, statusReq)
statusBody := framework.ReadAllAndClose(t, statusResp)
@@ -32,8 +34,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got)
}
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
t.Fatal("expected server-generated request id in response header")
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" {
t.Fatalf("expected echoed request id, got %q", got)
}
var payload map[string]interface{}
@@ -47,6 +49,7 @@ func TestAdminStatusAndHealthz(t *testing.T) {
}
healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz")
healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2")
healthResp := framework.DoRequest(t, client, healthReq)
_ = framework.ReadAllAndClose(t, healthResp)
if healthResp.StatusCode != http.StatusOK {
@@ -55,8 +58,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got)
}
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
t.Fatal("expected /healthz server-generated request id in response header")
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" {
t.Fatalf("expected /healthz echoed request id, got %q", got)
}
uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
-6
View File
@@ -80,11 +80,6 @@ type AccessKeyInfo struct {
CreatedAt time.Time `json:"created_at"`
}
type CreateAccessKeyRequest struct {
AccessKey string `json:"access_key"`
SecretKey string `json:"secret_key"`
}
type UpdateAccessKeyStatusRequest struct {
Status string `json:"status" binding:"required"`
}
@@ -95,7 +90,6 @@ type UserDetails struct {
Actions []string `json:"actions"`
PolicyNames []string `json:"policy_names"`
AccessKeys []AccessKeyInfo `json:"access_keys"`
Groups []string `json:"groups"`
}
type FilerNode struct {
-149
View File
@@ -1,149 +0,0 @@
package dash
import (
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
)
const (
adminLockName = cluster.AdminShellLockName
adminLockClientName = "admin-plugin"
)
// AdminLockManager coordinates exclusive admin locks with reference counting.
// It is safe for concurrent use.
type AdminLockManager struct {
locker *exclusive_locks.ExclusiveLocker
clientName string
mu sync.Mutex
cond *sync.Cond
acquiring bool
holdCount int
lastAcquiredAt time.Time
lastReleasedAt time.Time
waitingSince time.Time
waitingReason string
currentReason string
}
func NewAdminLockManager(masterClient *wdclient.MasterClient, clientName string) *AdminLockManager {
if masterClient == nil {
return nil
}
if clientName == "" {
clientName = adminLockClientName
}
manager := &AdminLockManager{
locker: exclusive_locks.NewExclusiveLocker(masterClient, adminLockName),
clientName: clientName,
}
manager.cond = sync.NewCond(&manager.mu)
return manager
}
func (m *AdminLockManager) Acquire(reason string) (func(), error) {
if m == nil || m.locker == nil {
return func() {}, nil
}
m.mu.Lock()
if reason != "" {
m.locker.SetMessage(reason)
m.currentReason = reason
}
for m.acquiring {
m.cond.Wait()
}
if m.holdCount == 0 {
m.acquiring = true
m.waitingSince = time.Now().UTC()
m.waitingReason = reason
m.mu.Unlock()
m.locker.RequestLock(m.clientName)
m.mu.Lock()
m.acquiring = false
m.holdCount = 1
m.lastAcquiredAt = time.Now().UTC()
m.waitingSince = time.Time{}
m.waitingReason = ""
m.cond.Broadcast()
m.mu.Unlock()
return m.Release, nil
}
m.holdCount++
if reason != "" {
m.currentReason = reason
}
m.mu.Unlock()
return m.Release, nil
}
func (m *AdminLockManager) Release() {
if m == nil || m.locker == nil {
return
}
m.mu.Lock()
if m.holdCount <= 0 {
m.mu.Unlock()
return
}
m.holdCount--
shouldRelease := m.holdCount == 0
m.mu.Unlock()
if shouldRelease {
m.mu.Lock()
m.lastReleasedAt = time.Now().UTC()
m.currentReason = ""
m.mu.Unlock()
m.locker.ReleaseLock()
}
}
type LockStatus struct {
Held bool `json:"held"`
HoldCount int `json:"hold_count"`
Acquiring bool `json:"acquiring"`
Message string `json:"message,omitempty"`
WaitingReason string `json:"waiting_reason,omitempty"`
LastAcquiredAt *time.Time `json:"last_acquired_at,omitempty"`
LastReleasedAt *time.Time `json:"last_released_at,omitempty"`
WaitingSince *time.Time `json:"waiting_since,omitempty"`
}
func (m *AdminLockManager) Status() LockStatus {
if m == nil {
return LockStatus{}
}
m.mu.Lock()
defer m.mu.Unlock()
status := LockStatus{
Held: m.holdCount > 0,
HoldCount: m.holdCount,
Acquiring: m.acquiring,
Message: m.currentReason,
WaitingReason: m.waitingReason,
}
if !m.lastAcquiredAt.IsZero() {
at := m.lastAcquiredAt
status.LastAcquiredAt = &at
}
if !m.lastReleasedAt.IsZero() {
at := m.lastReleasedAt
status.LastReleasedAt = &at
}
if !m.waitingSince.IsZero() {
at := m.waitingSince
status.WaitingSince = &at
}
return status
}
-61
View File
@@ -1,61 +0,0 @@
package dash
import (
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
)
const adminPresenceClientName = "admin-server"
type adminPresenceLock struct {
locker *exclusive_locks.ExclusiveLocker
stopCh chan struct{}
}
func newAdminPresenceLock(masterClient *wdclient.MasterClient) *adminPresenceLock {
if masterClient == nil {
return nil
}
return &adminPresenceLock{
locker: exclusive_locks.NewExclusiveLocker(masterClient, cluster.AdminServerPresenceLockName),
stopCh: make(chan struct{}),
}
}
func (l *adminPresenceLock) Start() {
if l == nil || l.locker == nil {
return
}
l.locker.SetMessage("admin server connected")
go func() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
if !l.locker.IsLocked() {
l.locker.RequestLock(adminPresenceClientName)
}
select {
case <-l.stopCh:
return
case <-ticker.C:
}
}
}()
}
func (l *adminPresenceLock) Stop() {
if l == nil {
return
}
select {
case <-l.stopCh:
default:
close(l.stopCh)
}
if l.locker != nil {
l.locker.ReleaseLock()
}
}
+35 -305
View File
@@ -2,18 +2,10 @@ package dash
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"sort"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/cluster"
clustermaintenance "github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -27,6 +19,10 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc"
"net/http"
"sort"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/s3api"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
@@ -101,9 +97,6 @@ type AdminServer struct {
// Maintenance system
maintenanceManager *maintenance.MaintenanceManager
plugin *adminplugin.Plugin
pluginLock *AdminLockManager
adminPresenceLock *adminPresenceLock
expireJobHandler func(jobID string, reason string) (*adminplugin.TrackedJob, bool, error)
// Topic retention purger
topicRetentionPurger *TopicRetentionPurger
@@ -140,12 +133,6 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
ctx := context.Background()
go masterClient.KeepConnectedToMaster(ctx)
lockManager := NewAdminLockManager(masterClient, adminLockClientName)
presenceLock := newAdminPresenceLock(masterClient)
if presenceLock != nil {
presenceLock.Start()
}
server := &AdminServer{
masterClient: masterClient,
templateFS: templateFS,
@@ -157,8 +144,6 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
collectionStatsCacheThreshold: defaultStatsCacheTimeout,
s3TablesManager: newS3TablesManager(),
icebergPort: icebergPort,
pluginLock: lockManager,
adminPresenceLock: presenceLock,
}
// Initialize topic retention purger
@@ -237,19 +222,20 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
}()
}
pluginOpts := adminplugin.Options{
plugin, err := adminplugin.New(adminplugin.Options{
DataDir: dataDir,
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
return server.buildDefaultPluginClusterContext(), nil
},
LockManager: lockManager,
ConfigDefaultsProvider: server.enrichConfigDefaults,
}
plugin, err := adminplugin.New(pluginOpts)
})
if err != nil && dataDir != "" {
glog.Warningf("Failed to initialize plugin with dataDir=%q: %v. Falling back to in-memory plugin state.", dataDir, err)
pluginOpts.DataDir = ""
plugin, err = adminplugin.New(pluginOpts)
plugin, err = adminplugin.New(adminplugin.Options{
DataDir: "",
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
return server.buildDefaultPluginClusterContext(), nil
},
})
}
if err != nil {
glog.Errorf("Failed to initialize plugin: %v", err)
@@ -273,89 +259,6 @@ func (s *AdminServer) loadTaskConfigurationsFromPersistence() {
configUpdateRegistry.UpdateAllConfigs(s.configPersistence)
}
// enrichConfigDefaults is called by the plugin when bootstrapping a job type's
// default config from its descriptor. For admin_script, it fetches maintenance
// scripts from the master and uses them as the script default.
//
// MIGRATION: This exists to help users migrate from master.toml [master.maintenance]
// to the admin script plugin worker. Remove after March 2027.
func (s *AdminServer) enrichConfigDefaults(cfg *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig {
if cfg.JobType != "admin_script" {
return cfg
}
var maintenanceScripts string
var sleepMinutes uint32
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return err
}
maintenanceScripts = resp.MaintenanceScripts
sleepMinutes = resp.MaintenanceSleepMinutes
return nil
})
if err != nil {
glog.V(1).Infof("Could not fetch master configuration for admin_script defaults: %v", err)
return cfg
}
script := cleanMaintenanceScript(maintenanceScripts)
if script == "" {
return cfg
}
interval := int64(sleepMinutes)
if interval <= 0 {
interval = clustermaintenance.DefaultMaintenanceSleepMinutes
}
glog.V(0).Infof("Enriching admin_script defaults from master maintenance scripts (interval=%dm)", interval)
if cfg.AdminConfigValues == nil {
cfg.AdminConfigValues = make(map[string]*plugin_pb.ConfigValue)
}
cfg.AdminConfigValues["script"] = &plugin_pb.ConfigValue{
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: script},
}
cfg.AdminConfigValues["run_interval_minutes"] = &plugin_pb.ConfigValue{
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: interval},
}
cfg.UpdatedBy = "master_migration"
return cfg
}
// cleanMaintenanceScript strips lock/unlock commands and normalizes a
// maintenance script string for use with the admin script plugin worker.
//
// MIGRATION: Used by enrichConfigDefaults. Remove after March 2027.
func cleanMaintenanceScript(script string) string {
script = strings.ReplaceAll(script, "\r\n", "\n")
var lines []string
for _, line := range strings.Split(script, "\n") {
trimmed := strings.TrimSpace(line)
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
continue
}
// Strip inline comments (e.g., "lock # migration note")
if idx := strings.Index(trimmed, "#"); idx >= 0 {
trimmed = strings.TrimSpace(trimmed[:idx])
if trimmed == "" {
continue
}
}
firstToken := strings.ToLower(strings.Fields(trimmed)[0])
if firstToken == "lock" || firstToken == "unlock" {
continue
}
lines = append(lines, trimmed)
}
return strings.Join(lines, "\n")
}
// GetCredentialManager returns the credential manager
func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
return s.credentialManager
@@ -373,21 +276,8 @@ func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
// InvalidateCache method moved to cluster_topology.go
// GetS3BucketsData retrieves Object Store buckets with pagination and sorting
func (s *AdminServer) GetS3BucketsData(page, pageSize int, sortBy, sortOrder string) (S3BucketsData, error) {
if page < 1 {
page = 1
}
if pageSize < 1 || pageSize > 1000 {
pageSize = 100
}
if sortBy == "" {
sortBy = "name"
}
if sortOrder == "" {
sortOrder = "asc"
}
// GetS3BucketsData retrieves all Object Store buckets and aggregates total storage metrics
func (s *AdminServer) GetS3BucketsData() (S3BucketsData, error) {
buckets, err := s.GetS3Buckets()
if err != nil {
return S3BucketsData{}, err
@@ -398,97 +288,14 @@ func (s *AdminServer) GetS3BucketsData(page, pageSize int, sortBy, sortOrder str
totalSize += bucket.PhysicalSize
}
totalBuckets := len(buckets)
// Sort buckets
s.sortBuckets(buckets, sortBy, sortOrder)
// Calculate pagination
totalPages := (totalBuckets + pageSize - 1) / pageSize
if totalPages == 0 {
totalPages = 1
}
if page > totalPages {
page = totalPages
}
startIndex := (page - 1) * pageSize
endIndex := startIndex + pageSize
if startIndex >= totalBuckets {
buckets = []S3Bucket{}
} else {
if endIndex > totalBuckets {
endIndex = totalBuckets
}
buckets = buckets[startIndex:endIndex]
}
return S3BucketsData{
Buckets: buckets,
TotalBuckets: totalBuckets,
TotalBuckets: len(buckets),
TotalSize: totalSize,
LastUpdated: time.Now(),
CurrentPage: page,
TotalPages: totalPages,
PageSize: pageSize,
SortBy: sortBy,
SortOrder: sortOrder,
}, nil
}
// sortBuckets sorts the bucket slice in place by the given field and order
func (s *AdminServer) sortBuckets(buckets []S3Bucket, sortBy, sortOrder string) {
desc := sortOrder == "desc"
sort.Slice(buckets, func(i, j int) bool {
a, b := buckets[i], buckets[j]
switch sortBy {
case "owner":
if a.Owner != b.Owner {
if desc {
return a.Owner > b.Owner
}
return a.Owner < b.Owner
}
case "created":
if !a.CreatedAt.Equal(b.CreatedAt) {
if desc {
return a.CreatedAt.After(b.CreatedAt)
}
return a.CreatedAt.Before(b.CreatedAt)
}
case "objects":
if a.ObjectCount != b.ObjectCount {
if desc {
return a.ObjectCount > b.ObjectCount
}
return a.ObjectCount < b.ObjectCount
}
case "logical_size":
if a.LogicalSize != b.LogicalSize {
if desc {
return a.LogicalSize > b.LogicalSize
}
return a.LogicalSize < b.LogicalSize
}
case "physical_size":
if a.PhysicalSize != b.PhysicalSize {
if desc {
return a.PhysicalSize > b.PhysicalSize
}
return a.PhysicalSize < b.PhysicalSize
}
}
// Tie-breaker: sort by name (also the default/primary for sortBy=="name")
if a.Name != b.Name {
if desc {
return a.Name > b.Name
}
return a.Name < b.Name
}
return false
})
}
// GetS3Buckets retrieves all Object Store buckets from the filer and collects size/object data from collections
func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
var buckets []S3Bucket
@@ -504,48 +311,28 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
// Now list buckets from the filer and match with collection data
err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// Paginate through all buckets in the buckets directory
const listPageSize = 1000
startFrom := ""
var snapshotTsNs int64
// List buckets by looking at the buckets directory
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: filerConfig.BucketsPath,
Prefix: "",
StartFromFileName: "",
InclusiveStartFrom: false,
Limit: 1000,
})
if err != nil {
return err
}
for {
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: filerConfig.BucketsPath,
Prefix: "",
StartFromFileName: startFrom,
InclusiveStartFrom: false,
Limit: listPageSize,
SnapshotTsNs: snapshotTsNs,
})
resp, err := stream.Recv()
if err != nil {
if err.Error() == "EOF" {
break
}
return err
}
pageCount := 0
lastName := ""
for {
resp, err := stream.Recv()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return err
}
if snapshotTsNs == 0 && resp.SnapshotTsNs != 0 {
snapshotTsNs = resp.SnapshotTsNs
}
if resp.Entry == nil {
continue
}
lastName = resp.Entry.Name
pageCount++
if !resp.Entry.IsDirectory {
continue
}
if resp.Entry.IsDirectory {
bucketName := resp.Entry.Name
if strings.HasPrefix(bucketName, ".") {
// Skip internal/system directories from Object Store bucket listing.
@@ -598,18 +385,13 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
}
}
var createdAt, lastModified time.Time
if resp.Entry.Attributes != nil {
createdAt = time.Unix(resp.Entry.Attributes.Crtime, 0)
lastModified = time.Unix(resp.Entry.Attributes.Mtime, 0)
}
bucket := S3Bucket{
Name: bucketName,
CreatedAt: createdAt,
CreatedAt: time.Unix(resp.Entry.Attributes.Crtime, 0),
LogicalSize: logicalSize,
PhysicalSize: physicalSize,
ObjectCount: objectCount,
LastModified: lastModified,
LastModified: time.Unix(resp.Entry.Attributes.Mtime, 0),
Quota: quota,
QuotaEnabled: quotaEnabled,
VersioningStatus: versioningStatus,
@@ -620,12 +402,6 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
}
buckets = append(buckets, bucket)
}
// If we received fewer entries than the page size, we've listed everything
if pageCount < listPageSize {
break
}
startFrom = lastName
}
return nil
@@ -1112,13 +888,6 @@ func (s *AdminServer) GetPlugin() *adminplugin.Plugin {
return s.plugin
}
func (s *AdminServer) acquirePluginLock(reason string) (func(), error) {
if s == nil || s.pluginLock == nil {
return func() {}, nil
}
return s.pluginLock.Acquire(reason)
}
// RequestPluginJobTypeDescriptor asks one worker for job type schema and returns the descriptor.
func (s *AdminServer) RequestPluginJobTypeDescriptor(ctx context.Context, jobType string, forceRefresh bool) (*plugin_pb.JobTypeDescriptor, error) {
if s.plugin == nil {
@@ -1161,13 +930,6 @@ func (s *AdminServer) RunPluginDetection(
if s.plugin == nil {
return nil, fmt.Errorf("plugin is not enabled")
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
if err != nil {
return nil, err
}
if releaseLock != nil {
defer releaseLock()
}
return s.plugin.RunDetection(ctx, jobType, clusterContext, maxResults)
}
@@ -1193,13 +955,6 @@ func (s *AdminServer) RunPluginDetectionWithReport(
if s.plugin == nil {
return nil, fmt.Errorf("plugin is not enabled")
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
if err != nil {
return nil, err
}
if releaseLock != nil {
defer releaseLock()
}
return s.plugin.RunDetectionWithReport(ctx, jobType, clusterContext, maxResults)
}
@@ -1213,17 +968,6 @@ func (s *AdminServer) ExecutePluginJob(
if s.plugin == nil {
return nil, fmt.Errorf("plugin is not enabled")
}
jobType := ""
if job != nil {
jobType = strings.TrimSpace(job.JobType)
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin execution %s", jobType))
if err != nil {
return nil, err
}
if releaseLock != nil {
defer releaseLock()
}
return s.plugin.ExecuteJob(ctx, job, clusterContext, attempt)
}
@@ -1236,7 +980,7 @@ func (s *AdminServer) GetPluginRunHistory(jobType string) (*adminplugin.JobTypeR
}
// ListPluginJobTypes returns known plugin job types from connected worker registry and persisted data.
func (s *AdminServer) ListPluginJobTypes() ([]adminplugin.JobTypeInfo, error) {
func (s *AdminServer) ListPluginJobTypes() ([]string, error) {
if s.plugin == nil {
return nil, fmt.Errorf("plugin is not enabled")
}
@@ -1275,17 +1019,6 @@ func (s *AdminServer) GetPluginJobDetail(jobID string, activityLimit, relatedLim
return s.plugin.BuildJobDetail(jobID, activityLimit, relatedLimit)
}
// ExpirePluginJob marks an active plugin job as failed so it no longer blocks scheduling.
func (s *AdminServer) ExpirePluginJob(jobID, reason string) (*adminplugin.TrackedJob, bool, error) {
if handler := s.expireJobHandler; handler != nil {
return handler(jobID, reason)
}
if s.plugin == nil {
return nil, false, fmt.Errorf("plugin is not enabled")
}
return s.plugin.ExpireJob(jobID, reason)
}
// ListPluginActivities returns plugin job activities for monitoring.
func (s *AdminServer) ListPluginActivities(jobType string, limit int) []adminplugin.JobActivity {
if s.plugin == nil {
@@ -1502,9 +1235,6 @@ func (s *AdminServer) Shutdown() {
// Stop maintenance manager
s.StopMaintenanceManager()
if s.adminPresenceLock != nil {
s.adminPresenceLock.Stop()
}
if s.plugin != nil {
s.plugin.Shutdown()

Some files were not shown because too many files have changed in this diff Show More