mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-12 17:45:53 +00:00
Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
feef8ed40e | ||
|
|
8e04f961f0 | ||
|
|
5400fdca3a | ||
|
|
d3d4e354bf | ||
|
|
df6f325574 | ||
|
|
0a6a76b71e | ||
|
|
93eff8fa45 | ||
|
|
cc53bb17ca | ||
|
|
e1f2e7e357 | ||
|
|
6e715f14d5 | ||
|
|
b09f4cc3e7 | ||
|
|
a8fab5503c |
@@ -32,7 +32,7 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
@@ -42,28 +42,28 @@ jobs:
|
||||
EOF
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -126,14 +126,14 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
- name: Set up QEMU
|
||||
if: matrix.platform != 'amd64'
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
@@ -106,25 +106,25 @@ jobs:
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Build ${{ matrix.platform }} ${{ matrix.variant }}
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
@@ -180,12 +180,12 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -35,14 +35,14 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v3
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
fi
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -60,16 +60,16 @@ jobs:
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
|
||||
# Full tags - multi-arch
|
||||
# Full tags - amd64 only
|
||||
- variant: full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
|
||||
# Large disk + full tags - amd64 only
|
||||
- variant: large_disk_full
|
||||
platforms: linux/amd64,linux/arm64
|
||||
platforms: linux/amd64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
@@ -117,7 +117,7 @@ jobs:
|
||||
|
||||
- name: Set up QEMU
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Create BuildKit config
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
@@ -129,20 +129,20 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
|
||||
- name: Build and push ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
@@ -198,14 +198,14 @@ jobs:
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -82,19 +82,19 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: true
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: '1.24'
|
||||
|
||||
- name: Build Telemetry Server
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.deploy
|
||||
|
||||
@@ -11,4 +11,4 @@ jobs:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48
|
||||
uses: actions/dependency-review-action@05fe4576374b728f0c523d6a13d64c25081e0803
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
@@ -135,7 +135,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: output-logs
|
||||
path: docker/output.log
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: ec-integration-test-logs
|
||||
path: |
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: '1.24'
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: ec-test-logs
|
||||
path: test/erasure_coding/admin_dockertest/tmp/logs/
|
||||
|
||||
@@ -22,6 +22,7 @@ permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '45m'
|
||||
|
||||
jobs:
|
||||
@@ -34,10 +35,10 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
|
||||
- name: Install FUSE and dependencies
|
||||
run: |
|
||||
@@ -182,7 +183,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: fuse-integration-test-results
|
||||
path: |
|
||||
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: test-reports-java-${{ matrix.java }}
|
||||
path: |
|
||||
|
||||
@@ -26,14 +26,14 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
@@ -132,7 +132,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -311,7 +311,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -473,7 +473,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -631,7 +631,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -789,7 +789,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: metadata-subscribe-test-logs
|
||||
path: |
|
||||
|
||||
@@ -20,6 +20,8 @@ jobs:
|
||||
include:
|
||||
- worker: erasure_coding
|
||||
path: test/plugin_workers/erasure_coding
|
||||
- worker: ec_repair
|
||||
path: test/plugin_workers/ec_repair
|
||||
- worker: vacuum
|
||||
path: test/plugin_workers/vacuum
|
||||
- worker: volume_balance
|
||||
|
||||
@@ -25,14 +25,14 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: postgres-logs
|
||||
path: test/postgres/postgres-output.log
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: integration-test-logs
|
||||
path: test/s3/normal/*.log
|
||||
|
||||
@@ -77,7 +77,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-filer-group-test-logs
|
||||
path: test/s3/filer_group/weed-test*.log
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-versioning-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-versioning-compatibility-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-cors-compatibility-logs
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -239,7 +239,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-retention-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -306,7 +306,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-cors-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -355,7 +355,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-retention-worm-logs
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -422,7 +422,7 @@ jobs:
|
||||
|
||||
- name: Upload stress test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-versioning-stress-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -478,7 +478,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-tagging-test-logs
|
||||
path: test/s3/tagging/weed-test*.log
|
||||
@@ -531,7 +531,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-remote-cache-test-logs
|
||||
path: |
|
||||
|
||||
@@ -5,8 +5,6 @@ on:
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/credential/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
push:
|
||||
@@ -14,8 +12,6 @@ on:
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/credential/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
|
||||
@@ -69,7 +65,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: iam-unit-test-results
|
||||
path: |
|
||||
@@ -84,7 +80,7 @@ jobs:
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group"]
|
||||
test-type: ["basic", "advanced", "policy-enforcement"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -121,7 +117,7 @@ jobs:
|
||||
"basic")
|
||||
echo "Running basic IAM functionality tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAMUserManagement|TestIAMAccessKeyManagement|TestIAMPolicyManagement" ./...
|
||||
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAM" ./...
|
||||
;;
|
||||
"advanced")
|
||||
echo "Running advanced IAM feature tests..."
|
||||
@@ -133,11 +129,6 @@ jobs:
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMPolicyEnforcement|TestS3IAMBucketPolicy|TestS3IAMContextual" ./...
|
||||
;;
|
||||
"group")
|
||||
echo "Running IAM group management tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestIAMGroup" ./...
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: ${{ matrix.test-type }}"
|
||||
exit 1
|
||||
@@ -171,7 +162,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-iam-integration-logs-${{ matrix.test-type }}
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -231,7 +222,7 @@ jobs:
|
||||
|
||||
- name: Upload distributed test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-iam-distributed-logs
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -283,7 +274,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-iam-performance-results
|
||||
path: |
|
||||
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-keycloak-test-logs
|
||||
path: |
|
||||
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: test-logs-python-${{ matrix.python-version }}
|
||||
path: |
|
||||
@@ -148,7 +148,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
cache: true
|
||||
|
||||
- name: Run Go unit tests
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: policy-unit-test-results
|
||||
path: |
|
||||
@@ -178,7 +178,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-policy-variables-test-logs
|
||||
path: /tmp/weed_policy_test_server.log
|
||||
@@ -299,7 +299,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
|
||||
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
|
||||
@@ -386,7 +386,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: trusted-proxy-test-logs
|
||||
path: /tmp/weed_proxy_test.log
|
||||
|
||||
@@ -1,122 +0,0 @@
|
||||
name: "S3 Proxy Signature Tests"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
proxy-signature-tests:
|
||||
name: S3 Proxy Signature Verification Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Build SeaweedFS binary for Linux
|
||||
run: |
|
||||
set -x
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -buildvcs=false -v -o test/s3/proxy_signature/weed ./weed
|
||||
|
||||
- name: Run S3 Proxy Signature Tests
|
||||
timeout-minutes: 10
|
||||
working-directory: test/s3/proxy_signature
|
||||
run: |
|
||||
set -x
|
||||
echo "Starting Docker Compose services..."
|
||||
docker compose up -d --build
|
||||
|
||||
# Check if containers are running
|
||||
echo "Checking container status..."
|
||||
docker compose ps
|
||||
|
||||
# Wait for services to be ready
|
||||
echo "Waiting for nginx proxy to be ready..."
|
||||
PROXY_READY=0
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s http://localhost:9000/ > /dev/null 2>&1; then
|
||||
echo "Proxy is ready"
|
||||
PROXY_READY=1
|
||||
break
|
||||
fi
|
||||
echo "Waiting for proxy... ($i/30)"
|
||||
sleep 1
|
||||
done
|
||||
if [ $PROXY_READY -eq 0 ]; then
|
||||
echo "ERROR: Proxy failed to become ready after 30 seconds"
|
||||
echo "Docker compose logs:"
|
||||
docker compose logs --no-color || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Wait for SeaweedFS to be ready
|
||||
echo "Waiting for SeaweedFS S3 gateway to be ready via proxy..."
|
||||
S3_READY=0
|
||||
for i in $(seq 1 30); do
|
||||
# Check logs first for startup message (weed mini says "S3 service is ready")
|
||||
if docker compose logs seaweedfs 2>&1 | grep -qE "S3 (gateway|service).*(started|ready)"; then
|
||||
echo "SeaweedFS S3 gateway is ready"
|
||||
S3_READY=1
|
||||
break
|
||||
fi
|
||||
# Fallback: check headers via proxy (which is already ready)
|
||||
if curl -s -I http://localhost:9000/ | grep -qi "SeaweedFS"; then
|
||||
echo "SeaweedFS S3 gateway is responding via proxy"
|
||||
S3_READY=1
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 gateway... ($i/30)"
|
||||
sleep 1
|
||||
done
|
||||
if [ $S3_READY -eq 0 ]; then
|
||||
echo "ERROR: SeaweedFS S3 gateway failed to become ready after 30 seconds"
|
||||
echo "Latest seaweedfs logs:"
|
||||
docker compose logs --no-color --tail 20 seaweedfs || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run the test script inside AWS CLI container
|
||||
echo "Running test script..."
|
||||
docker run --rm --network host \
|
||||
--entrypoint bash \
|
||||
amazon/aws-cli:latest \
|
||||
-c "$(cat test.sh)"
|
||||
|
||||
TEST_RESULT=$?
|
||||
|
||||
# Cleanup
|
||||
docker compose down
|
||||
|
||||
exit $TEST_RESULT
|
||||
|
||||
- name: Cleanup on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/proxy_signature
|
||||
run: |
|
||||
echo "Cleaning up Docker containers..."
|
||||
ls -al weed || true
|
||||
ldd weed || true
|
||||
echo "Docker compose logs:"
|
||||
docker compose logs --no-color || true
|
||||
echo "Container status before cleanup:"
|
||||
docker ps -a
|
||||
echo "Stopping services..."
|
||||
docker compose down || true
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-spark-test-logs
|
||||
path: test/s3/spark/test-output.log
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -143,7 +143,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-compatibility-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-metadata-persistence-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -241,7 +241,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-copy-operations-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -290,7 +290,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-multipart-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -340,7 +340,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-sse-performance-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -389,7 +389,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-volume-encryption-logs
|
||||
path: /tmp/seaweedfs-sse-*.log
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-tables-test-logs
|
||||
path: test/s3tables/table-buckets/test-output.log
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog/test-output.log
|
||||
@@ -144,7 +144,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Pre-pull Trino image
|
||||
run: docker pull trinodb/trino:479
|
||||
@@ -188,73 +188,12 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: trino-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_trino/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
polaris-integration-tests:
|
||||
name: Polaris Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Pre-pull Polaris image
|
||||
run: docker pull apache/polaris:latest
|
||||
|
||||
- name: Run Polaris Integration Tests
|
||||
timeout-minutes: 25
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Polaris Tests ==="
|
||||
|
||||
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
|
||||
echo "Polaris integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/polaris
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: polaris-test-logs
|
||||
path: test/s3tables/polaris/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
spark-iceberg-catalog-tests:
|
||||
name: Spark Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
@@ -271,7 +210,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Pre-pull Spark image
|
||||
run: docker pull apache/spark:3.5.1
|
||||
@@ -315,7 +254,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: spark-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_spark/test-output.log
|
||||
@@ -337,7 +276,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Pre-pull RisingWave image
|
||||
run: |
|
||||
@@ -383,7 +322,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: risingwave-catalog-test-logs
|
||||
path: test/s3tables/catalog_risingwave/test-output.log
|
||||
@@ -405,7 +344,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Pre-pull Python image
|
||||
run: docker pull python:3
|
||||
@@ -449,7 +388,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: sts-integration-test-logs
|
||||
path: test/s3tables/sts_integration/test-output.log
|
||||
@@ -471,7 +410,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Pre-pull Python image
|
||||
run: docker pull python:3
|
||||
@@ -518,7 +457,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: lakekeeper-integration-test-logs
|
||||
path: test/s3tables/lakekeeper/test-output.log
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
go-version: '1.24'
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
@@ -125,7 +125,7 @@ jobs:
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: spark-test-results
|
||||
path: test/java/spark/target/surefire-reports/
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.25
|
||||
go-version: ^1.24
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: seaweedfs-logs
|
||||
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tus-test-logs
|
||||
path: |
|
||||
|
||||
@@ -90,7 +90,7 @@ jobs:
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: volume-server-integration-test-logs
|
||||
path: /tmp/volume-server-it-logs/
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25 AS builder
|
||||
FROM golang:1.24 AS builder
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y build-essential wget ca-certificates && \
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.24-alpine as builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25 AS builder
|
||||
FROM golang:1.24 AS builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25 AS builder
|
||||
FROM golang:1.24 AS builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
@@ -7,7 +7,6 @@
|
||||
|
||||
[master.maintenance]
|
||||
# periodically run these scripts are the same as running them from 'weed shell'
|
||||
# Scripts are skipped while an admin server is connected.
|
||||
scripts = """
|
||||
lock
|
||||
ec.encode -fullPercent=95 -quietFor=1h
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.25.0
|
||||
go 1.24.9
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
@@ -26,7 +26,7 @@ require (
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.16.0
|
||||
github.com/go-redsync/redsync/v4 v4.15.0
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/golang/protobuf v1.5.4
|
||||
@@ -63,7 +63,7 @@ require (
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.2 // indirect
|
||||
github.com/prometheus/procfs v0.20.1
|
||||
github.com/prometheus/procfs v0.19.2
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
@@ -87,22 +87,22 @@ require (
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.6.7
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.mongodb.org/mongo-driver v1.17.6
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.45.0
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0
|
||||
gocloud.dev v0.44.0
|
||||
gocloud.dev/pubsub/natspubsub v0.44.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.44.0
|
||||
golang.org/x/crypto v0.48.0
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
|
||||
golang.org/x/image v0.36.0
|
||||
golang.org/x/net v0.49.0
|
||||
golang.org/x/oauth2 v0.35.0
|
||||
golang.org/x/sys v0.42.0
|
||||
golang.org/x/oauth2 v0.34.0
|
||||
golang.org/x/sys v0.41.0
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/tools v0.41.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.258.0
|
||||
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 // indirect
|
||||
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
|
||||
google.golang.org/grpc v1.78.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
@@ -124,13 +124,13 @@ require (
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.3
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.7
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.43.0
|
||||
github.com/getsentry/sentry-go v0.42.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.12
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
@@ -138,7 +138,8 @@ require (
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/lib/pq v1.11.1
|
||||
github.com/linkedin/goavro/v2 v2.14.1
|
||||
github.com/mattn/go-sqlite3 v1.14.34
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
@@ -150,7 +151,7 @@ require (
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1
|
||||
github.com/shirou/gopsutil/v4 v4.26.2
|
||||
github.com/shirou/gopsutil/v4 v4.26.1
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.1
|
||||
github.com/testcontainers/testcontainers-go v0.39.0
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
@@ -171,7 +172,7 @@ require (
|
||||
atomicgo.dev/keyboard v0.2.9 // indirect
|
||||
atomicgo.dev/schedule v0.1.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.7.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
@@ -236,7 +237,6 @@ require (
|
||||
github.com/klauspost/asmfmt v1.3.2 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lib/pq v1.11.1 // indirect
|
||||
github.com/lithammer/fuzzysearch v1.1.8 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
@@ -279,10 +279,10 @@ require (
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
@@ -292,12 +292,12 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cel.dev/expr v0.24.0 // indirect
|
||||
cloud.google.com/go/auth v0.17.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.2 // indirect
|
||||
filippo.io/edwards25519 v1.1.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
@@ -327,7 +327,7 @@ require (
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
@@ -336,12 +336,12 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6
|
||||
github.com/aws/smithy-go v1.24.2
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
|
||||
github.com/aws/smithy-go v1.24.0
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
@@ -352,7 +352,7 @@ require (
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
@@ -360,11 +360,11 @@ require (
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/ebitengine/purego v0.9.1 // indirect
|
||||
github.com/elastic/gosigar v0.14.3 // indirect
|
||||
github.com/emersion/go-message v0.18.2 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
@@ -428,8 +428,8 @@ require (
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nats.go v1.48.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.12 // indirect
|
||||
github.com/nats-io/nats.go v1.43.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.11 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.5 // indirect
|
||||
@@ -493,11 +493,11 @@ require (
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.40.0 // indirect
|
||||
|
||||
@@ -6,8 +6,8 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8=
|
||||
atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ=
|
||||
atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs=
|
||||
atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU=
|
||||
cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
|
||||
cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
|
||||
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
|
||||
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
|
||||
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
|
||||
cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
|
||||
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
|
||||
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
||||
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
||||
cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc=
|
||||
cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA=
|
||||
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
|
||||
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
|
||||
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
|
||||
@@ -338,8 +338,8 @@ cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJP
|
||||
cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk=
|
||||
cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4=
|
||||
cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w=
|
||||
cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
|
||||
cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
|
||||
cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM=
|
||||
cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U=
|
||||
cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA=
|
||||
cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o=
|
||||
cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM=
|
||||
@@ -393,8 +393,8 @@ cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhz
|
||||
cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
|
||||
cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM=
|
||||
cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk=
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0 h1:DgAN907x+sP0nScYfBzneRiIhWoXcpCD8ZAut8WX9vs=
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
|
||||
cloud.google.com/go/pubsub/v2 v2.2.1 h1:3brZcshL3fIiD1qOxAE2QW9wxsfjioy014x4yC9XuYI=
|
||||
cloud.google.com/go/pubsub/v2 v2.2.1/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
|
||||
cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
|
||||
cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
|
||||
cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
|
||||
@@ -504,8 +504,8 @@ cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV6
|
||||
cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28=
|
||||
cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y=
|
||||
cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA=
|
||||
cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
|
||||
cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
|
||||
cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4=
|
||||
cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI=
|
||||
cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs=
|
||||
cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg=
|
||||
cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos=
|
||||
@@ -687,8 +687,6 @@ github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUS
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op h1:Ucf+QxEKMbPogRO5guBNe5cgd9uZgfoJLOYs8WWhtjM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
|
||||
github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4=
|
||||
@@ -716,8 +714,8 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
|
||||
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
|
||||
@@ -726,8 +724,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUT
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 h1:2fjfz3/G9BRvIKuNZ655GwzpklC2kEH0cowZQGO7uBg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4/go.mod h1:Ymws824lvMypLFPwyyUXM52SXuGgxpu0+DISLfKvB+c=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
|
||||
@@ -748,18 +746,18 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZi
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
@@ -857,8 +855,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
|
||||
github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e h1:gt7U1Igw0xbJdyaCM5H2CnlAlPSkzrhsebQB6WQWjLA=
|
||||
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
|
||||
github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
|
||||
github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
|
||||
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
|
||||
@@ -974,8 +972,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
|
||||
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
|
||||
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJtLmY22n99HaZTz+r2Z51xUPi01m3wg=
|
||||
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg=
|
||||
github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
|
||||
@@ -1002,8 +1000,8 @@ github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJ
|
||||
github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
|
||||
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
|
||||
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
@@ -1053,8 +1051,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
|
||||
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
|
||||
github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
|
||||
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
|
||||
@@ -1126,8 +1124,8 @@ github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOr
|
||||
github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-redsync/redsync/v4 v4.16.0 h1:bNcOzeHH9d3s6pghU9NJFMPrQa41f5Nx3L4YKr3BdEU=
|
||||
github.com/go-redsync/redsync/v4 v4.16.0/go.mod h1:V4gagqgyASWBZuwx4xGzu72aZNb/6Mo05byUa3mVmKQ=
|
||||
github.com/go-redsync/redsync/v4 v4.15.0 h1:KH/XymuxSV7vyKs6z1Cxxj+N+N18JlPxgXeP6x4JY54=
|
||||
github.com/go-redsync/redsync/v4 v4.15.0/go.mod h1:qNp+lLs3vkfZbtA/aM/OjlZHfEr5YTAYhRktFPKHC7s=
|
||||
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
|
||||
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
@@ -1239,8 +1237,6 @@ github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV
|
||||
github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI=
|
||||
github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
|
||||
github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
|
||||
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
@@ -1518,8 +1514,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
|
||||
github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
|
||||
github.com/linkedin/goavro/v2 v2.15.0 h1:pDj1UrjUOO62iXhgBiE7jQkpNIc5/tA5eZsgolMjgVI=
|
||||
github.com/linkedin/goavro/v2 v2.15.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
|
||||
github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkFS2iI=
|
||||
github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
|
||||
github.com/linxGnu/grocksdb v1.10.7 h1:fCi4qvZWo04VgFwGWmO8HQJgUVounJBy+C2TMVPU/ho=
|
||||
github.com/linxGnu/grocksdb v1.10.7/go.mod h1:OLQKZwiKwaJiAVCsOzWKvwiLwfZ5Vz8Md5TYR7t7pM8=
|
||||
github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=
|
||||
@@ -1567,8 +1563,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
|
||||
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
|
||||
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
@@ -1626,14 +1622,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
|
||||
github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
|
||||
github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
|
||||
github.com/nats-io/nats-server/v2 v2.11.12 h1:jGDXTkcjqQ5fCRstwIxvv1K0RHfftFUoSCT/iIZcqOc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.12/go.mod h1:5MCp/pqm5SEfsvVZ31ll1088ZTwEUdvRX1Hmh/mTTDg=
|
||||
github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U=
|
||||
github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
|
||||
github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
|
||||
github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
|
||||
github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak=
|
||||
github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
|
||||
github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU=
|
||||
github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0=
|
||||
github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug=
|
||||
github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
|
||||
github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
|
||||
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
@@ -1769,8 +1765,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
|
||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
||||
github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI=
|
||||
github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg=
|
||||
github.com/pterm/pterm v0.12.30/go.mod h1:MOqLIyMOgmTDz9yorcYbcw+HsgoZo3BQfg2wtl3HEFE=
|
||||
@@ -1804,10 +1800,10 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/redis/rueidis v1.0.71 h1:pODtnAR5GAB7j4ekhldZ29HKOxe4Hph0GTDGk1ayEQY=
|
||||
github.com/redis/rueidis v1.0.71/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.71 h1:wNZ//kEjMZgBM0KCk7ncOX8KmAgROU2kDdDNpwheG4w=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.71/go.mod h1:esmCLJvaRzZoKlgB82G1bY7Iky5TnO9Rz+NlhbEccFI=
|
||||
github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
|
||||
github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.69/go.mod h1:iC4Y8DoN0Uth0Uezg9e2trvNRC7QAgGeuP2OPLb5ccI=
|
||||
github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
|
||||
github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM=
|
||||
github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
|
||||
@@ -1861,8 +1857,8 @@ github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQ
|
||||
github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
|
||||
github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
|
||||
github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/shirou/gopsutil/v4 v4.26.1 h1:TOkEyriIXk2HX9d4isZJtbjXbEjf5qyKPAzbzY0JWSo=
|
||||
github.com/shirou/gopsutil/v4 v4.26.1/go.mod h1:medLI9/UNAb0dOI9Q3/7yWSqKkj00u+1tgY8nvv41pc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
|
||||
@@ -2113,8 +2109,8 @@ go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/Ah
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
|
||||
go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
@@ -2134,35 +2130,35 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
|
||||
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
|
||||
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0 h1:zG8GlgXCJQd5BU98C0hZnBbElszTmUgCNCfYneaDL0A=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0/go.mod h1:hOfBCz8kv/wuq73Mx2H2QnWokh/kHZxkh6SNF2bdKtw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 h1:ZsXq73BERAiNuuFXYqP4MR5hBrjXfMGSO+Cx7qoOZiM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0/go.mod h1:hg1zaDMpyZJuUzjFxFsRYBoccE86tM9Uf4IqNMUxvrY=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
|
||||
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
|
||||
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
|
||||
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
|
||||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
@@ -2186,10 +2182,10 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY=
|
||||
gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI=
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0 h1:kfCupVejeynIQcS1GaIvkY3Nj/acLlM5yPWyxZN8wJ8=
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0/go.mod h1:WU5SMDWuF7CCr2U8UpbrEONYYFOmvigCAvTpOOpUvYE=
|
||||
gocloud.dev v0.44.0 h1:iVyMAqFl2r6xUy7M4mfqwlN+21UpJoEtgHEcfiLMUXs=
|
||||
gocloud.dev v0.44.0/go.mod h1:ZmjROXGdC/eKZLF1N+RujDlFRx3D+4Av2thREKDMVxY=
|
||||
gocloud.dev/pubsub/natspubsub v0.44.0 h1:1Us76ckkdgtiE1p1rJZ+38b9TQP051bmjAiQlFQzYrM=
|
||||
gocloud.dev/pubsub/natspubsub v0.44.0/go.mod h1:PvVAGIhL14PWGwWIXX/zAK42ixr2/PKP4Q4yMiAUraQ=
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.44.0 h1:MpRIO6XJ/JTqrlUWt3CxwDe1LvaiXUVu4sS5cv4f/AM=
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.44.0/go.mod h1:BB9+qT3r6g4M5+4asiXaEeqw4QAOzsWusO5krYaqkdA=
|
||||
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
|
||||
@@ -2380,8 +2376,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
|
||||
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
|
||||
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
|
||||
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
|
||||
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
|
||||
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
|
||||
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -2509,8 +2505,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 h1:O1cMQHRfwNpDfDJerqRoE2oD+AFlyid87D40L/OkkJo=
|
||||
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8=
|
||||
@@ -2844,8 +2840,8 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
|
||||
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
|
||||
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
|
||||
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
|
||||
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
|
||||
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
|
||||
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc=
|
||||
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 h1:ZdyUkS9po3H7G0tuh955QVyyotWvOD4W0aEapeGeUYk=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846/go.mod h1:Fk4kyraUvqD7i5H6S43sj2W98fbZa75lpZz/eUyhfO0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.16"
|
||||
appVersion: "4.13"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.16.0
|
||||
version: 4.0.413
|
||||
|
||||
@@ -23,15 +23,8 @@
|
||||
#
|
||||
# Adjust storageClass and sizes to match your cluster's available StorageClasses.
|
||||
# On OpenShift you can discover them with: oc get storageclass
|
||||
global:
|
||||
enableReplication: true
|
||||
# replication type is XYZ:
|
||||
# X number of replica in other data centers
|
||||
# Y number of replica in other racks in the same data center
|
||||
# Z number of replica in other servers in the same rack
|
||||
replicationPlacement: "000" # no data replica
|
||||
|
||||
master:
|
||||
replicas: 1
|
||||
data:
|
||||
type: "persistentVolumeClaim"
|
||||
size: "10Gi"
|
||||
@@ -56,7 +49,6 @@ master:
|
||||
type: RuntimeDefault
|
||||
|
||||
volume:
|
||||
replicas: 1
|
||||
dataDirs:
|
||||
- name: data1
|
||||
type: "persistentVolumeClaim"
|
||||
@@ -83,7 +75,6 @@ volume:
|
||||
type: RuntimeDefault
|
||||
|
||||
filer:
|
||||
replicas: 1
|
||||
data:
|
||||
type: "persistentVolumeClaim"
|
||||
size: "25Gi"
|
||||
@@ -109,19 +100,12 @@ filer:
|
||||
|
||||
# S3 gateway (if enabled)
|
||||
s3:
|
||||
enabled: true
|
||||
replicas: 1
|
||||
port: 8333
|
||||
enableAuth: true
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
|
||||
# controller assign them automatically based on the namespace's SCC.
|
||||
runAsNonRoot: true
|
||||
|
||||
logs:
|
||||
type: "emptyDir"
|
||||
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -118,10 +118,8 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.admin "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
{{- if .Values.admin.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.admin.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -130,6 +128,18 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
|
||||
@@ -243,7 +243,8 @@ spec:
|
||||
{{- if $httpsPort }}
|
||||
-s3.port.https={{ $httpsPort }} \
|
||||
{{- end }}
|
||||
{{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
-s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{- end }}
|
||||
{{- if or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/s3/seaweedfs_s3_config \
|
||||
@@ -345,9 +346,6 @@ spec:
|
||||
- name: client-cert
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
readOnly: true
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
{{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.allInOne.extraVolumeMounts . | nindent 12 }}
|
||||
ports:
|
||||
@@ -475,9 +473,6 @@ spec:
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-client-cert
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
{{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.allInOne.extraVolumes . | nindent 8 }}
|
||||
{{- if .Values.allInOne.nodeSelector }}
|
||||
|
||||
@@ -17,9 +17,6 @@ metadata:
|
||||
spec:
|
||||
type: {{ .Values.allInOne.service.type | default "ClusterIP" }}
|
||||
internalTrafficPolicy: {{ .Values.allInOne.service.internalTrafficPolicy | default "Cluster" }}
|
||||
{{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) .Values.allInOne.s3.trafficDistribution }}
|
||||
trafficDistribution: {{ include "seaweedfs.trafficDistribution" (dict "value" .Values.allInOne.s3.trafficDistribution "Capabilities" .Capabilities) }}
|
||||
{{- end }}
|
||||
ports:
|
||||
# Master ports
|
||||
- name: "swfs-master"
|
||||
|
||||
@@ -6,10 +6,6 @@ metadata:
|
||||
name: {{ .Values.cosi.bucketClassName }}
|
||||
driverName: {{ .Values.cosi.driverName }}
|
||||
deletionPolicy: Delete
|
||||
{{- with .Values.cosi.bucketClassParameters }}
|
||||
parameters:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
---
|
||||
kind: BucketAccessClass
|
||||
apiVersion: objectstorage.k8s.io/v1alpha1
|
||||
|
||||
@@ -96,10 +96,8 @@ spec:
|
||||
- name: WEED_GRPC_CA
|
||||
value: /usr/local/share/ca-certificates/client/ca.crt
|
||||
{{- end }}
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.cosi "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
{{- if .Values.cosi.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.cosi.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -108,6 +106,18 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- mountPath: /var/lib/cosi
|
||||
name: socket
|
||||
|
||||
@@ -114,10 +114,8 @@ spec:
|
||||
optional: true
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.filer "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
{{- if .Values.filer.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.filer.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -126,6 +124,18 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.secretExtraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
@@ -200,7 +210,8 @@ spec:
|
||||
{{- if .Values.filer.s3.httpsPort }}
|
||||
-s3.port.https={{ .Values.filer.s3.httpsPort }} \
|
||||
{{- end }}
|
||||
{{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
-s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{- end }}
|
||||
{{- if .Values.filer.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/seaweedfs_s3_config \
|
||||
@@ -253,9 +264,6 @@ spec:
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
{{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }}
|
||||
ports:
|
||||
@@ -386,9 +394,6 @@ spec:
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-client-cert
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
{{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.filer.extraVolumes . | indent 8 | trim }}
|
||||
{{- if .Values.filer.nodeSelector }}
|
||||
|
||||
@@ -69,7 +69,9 @@ spec:
|
||||
priorityClassName: {{ .Values.master.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
{{- if .Values.global.createClusterRole }}
|
||||
serviceAccountName: {{ .Values.master.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
|
||||
{{- end }}
|
||||
{{- if .Values.master.initContainers }}
|
||||
initContainers:
|
||||
{{ tpl .Values.master.initContainers . | nindent 8 | trim }}
|
||||
@@ -96,10 +98,8 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.master "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
{{- if .Values.master.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.master.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -108,6 +108,18 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
|
||||
@@ -90,10 +90,8 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.s3 "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
{{- if .Values.s3.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.s3.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -102,6 +100,18 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
@@ -127,7 +137,8 @@ spec:
|
||||
{{- if .Values.s3.httpsPort }}
|
||||
-port.https={{ .Values.s3.httpsPort }} \
|
||||
{{- end }}
|
||||
{{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
|
||||
-cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{- end }}
|
||||
{{- if .Values.s3.domainName }}
|
||||
-domainName={{ .Values.s3.domainName }} \
|
||||
@@ -175,7 +186,6 @@ spec:
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
{{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.s3.extraVolumeMounts . | nindent 12 | trim }}
|
||||
ports:
|
||||
@@ -267,7 +277,6 @@ spec:
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-client-cert
|
||||
{{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.s3.extraVolumes . | indent 8 | trim }}
|
||||
{{- if .Values.s3.nodeSelector }}
|
||||
|
||||
@@ -16,9 +16,8 @@ metadata:
|
||||
{{- end }}
|
||||
spec:
|
||||
internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }}
|
||||
{{- $td := .Values.s3.trafficDistribution | default .Values.filer.s3.trafficDistribution }}
|
||||
{{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) $td }}
|
||||
trafficDistribution: {{ include "seaweedfs.trafficDistribution" (dict "value" $td "Capabilities" .Capabilities) }}
|
||||
{{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) (or .Values.s3.trafficDistribution .Values.filer.s3.trafficDistribution) }}
|
||||
trafficDistribution: {{ include "seaweedfs.trafficDistribution" . }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: "swfs-s3"
|
||||
|
||||
@@ -90,10 +90,8 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.sftp "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
{{- if .Values.sftp.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.sftp.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -102,6 +100,18 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
|
||||
@@ -59,18 +59,6 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "seaweedfs.mergeExtraEnvironmentVars" -}}
|
||||
{{- $global := ((.global | default dict).extraEnvironmentVars | default dict) -}}
|
||||
{{- $component := ((.component | default dict).extraEnvironmentVars | default dict) -}}
|
||||
{{- $target := .target -}}
|
||||
{{- range $key, $value := $global }}
|
||||
{{- $_ := set $target $key $value }}
|
||||
{{- end }}
|
||||
{{- range $key, $value := $component }}
|
||||
{{- $_ := set $target $key $value }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper filer image */}}
|
||||
{{- define "filer.image" -}}
|
||||
{{- if .Values.filer.imageOverride -}}
|
||||
@@ -338,41 +326,11 @@ Create the name of the service account to use
|
||||
{{- .Values.global.serviceAccountName | default "seaweedfs" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* S3 TLS cert/key arguments, using custom secret if s3.tlsSecret is set */}}
|
||||
{{- define "seaweedfs.s3.tlsArgs" -}}
|
||||
{{- $prefix := .prefix -}}
|
||||
{{- $root := .root -}}
|
||||
{{- if $root.Values.s3.tlsSecret -}}
|
||||
-{{ $prefix }}cert.file=/usr/local/share/ca-certificates/s3/tls.crt \
|
||||
-{{ $prefix }}key.file=/usr/local/share/ca-certificates/s3/tls.key \
|
||||
{{- else -}}
|
||||
-{{ $prefix }}cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-{{ $prefix }}key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* S3 custom TLS volume mount */}}
|
||||
{{- define "seaweedfs.s3.tlsVolumeMount" -}}
|
||||
{{- if .Values.s3.tlsSecret }}
|
||||
- name: s3-tls-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/s3/
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* S3 custom TLS volume */}}
|
||||
{{- define "seaweedfs.s3.tlsVolume" -}}
|
||||
{{- if .Values.s3.tlsSecret }}
|
||||
- name: s3-tls-cert
|
||||
secret:
|
||||
secretName: {{ .Values.s3.tlsSecret }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Generate a compatible trafficDistribution value due to "PreferClose" fast deprecation in k8s v1.35.
|
||||
Accepts a dict with "value" (the trafficDistribution string) and "Capabilities". */}}
|
||||
{{/* Generate a compatible trafficDistribution value due to "PreferClose" fast deprecation in k8s v1.35 */}}
|
||||
{{- define "seaweedfs.trafficDistribution" -}}
|
||||
{{- if .value -}}
|
||||
{{- and (eq .value "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .value -}}
|
||||
{{- if .Values.s3.trafficDistribution -}}
|
||||
{{- and (eq .Values.s3.trafficDistribution "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .Values.s3.trafficDistribution -}}
|
||||
{{- else if .Values.filer.s3.trafficDistribution -}}
|
||||
{{- and (eq .Values.filer.s3.trafficDistribution "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .Values.filer.s3.trafficDistribution -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
@@ -3,15 +3,6 @@
|
||||
{{- $s3Enabled := false }}
|
||||
{{- $enableAuth := false }}
|
||||
{{- $existingConfigSecret := "" }}
|
||||
{{- $bucketsFolder := "/buckets" }}
|
||||
{{- $bucketEnvVars := merge (dict) (.Values.global.extraEnvironmentVars | default dict) }}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
{{- $bucketEnvVars = merge (.Values.allInOne.extraEnvironmentVars | default dict) $bucketEnvVars }}
|
||||
{{- else }}
|
||||
{{- $bucketEnvVars = merge (.Values.filer.extraEnvironmentVars | default dict) $bucketEnvVars }}
|
||||
{{- end }}
|
||||
{{- $bucketsFolder = default $bucketsFolder (get $bucketEnvVars "WEED_FILER_BUCKETS_FOLDER") }}
|
||||
{{- $bucketsFolder = trimSuffix "/" $bucketsFolder }}
|
||||
|
||||
{{- /* Check allInOne mode first */}}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
@@ -51,7 +42,7 @@ metadata:
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "-5"
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
"helm.sh/hook-delete-policy": hook-succeeded
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
@@ -92,7 +83,6 @@ spec:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
- |
|
||||
set -o pipefail
|
||||
wait_for_service() {
|
||||
local url=$1
|
||||
local max_attempts=60 # 5 minutes total (5s * 60)
|
||||
@@ -118,51 +108,15 @@ spec:
|
||||
wait_for_service "http://$WEED_CLUSTER_SW_MASTER{{ .Values.master.readinessProbe.httpGet.path }}"
|
||||
wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}"
|
||||
{{- end }}
|
||||
{{- range $createBuckets }}
|
||||
{{- $bucketName := .name }}
|
||||
{{- $bucketLock := or .lock .objectLock .withLock }}
|
||||
bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }
|
||||
if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then
|
||||
echo "Bucket '{{ $bucketName }}' already exists, skipping creation."
|
||||
{{- range $createBuckets }}
|
||||
if /bin/echo "s3.bucket.list" | /usr/bin/weed shell | awk '{print $1}' | grep -Fxq "{{ .name }}"; then
|
||||
echo "Bucket '{{ .name }}' already exists, skipping creation."
|
||||
else
|
||||
echo "Creating bucket '{{ $bucketName }}'..."
|
||||
/bin/echo 's3.bucket.create --name {{ $bucketName }}{{- if $bucketLock }} --withLock{{- end }}' | /usr/bin/weed shell
|
||||
echo "Creating bucket '{{ .name }}'..."
|
||||
/bin/echo "s3.bucket.create --name {{ .name }}" | /usr/bin/weed shell
|
||||
fi
|
||||
{{- end }}
|
||||
{{- range $createBuckets }}
|
||||
{{- $bucketLock := or .lock .objectLock .withLock }}
|
||||
{{- if $bucketLock }}
|
||||
/bin/echo 's3.bucket.lock -name {{ .name }} -enable' | /usr/bin/weed shell
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range $createBuckets }}
|
||||
{{- $bucketVersioning := "" }}
|
||||
{{- if kindIs "bool" .versioning }}
|
||||
{{- if .versioning }}
|
||||
{{- $bucketVersioning = "Enabled" }}
|
||||
{{- end }}
|
||||
{{- else if kindIs "string" .versioning }}
|
||||
{{- $versioningLower := lower .versioning }}
|
||||
{{- if eq $versioningLower "enabled" "enable" "true" }}
|
||||
{{- $bucketVersioning = "Enabled" }}
|
||||
{{- else if eq $versioningLower "suspended" "disable" "false" }}
|
||||
{{- $bucketVersioning = "Suspended" }}
|
||||
{{- else if or (eq .versioning "Enabled") (eq .versioning "Suspended") }}
|
||||
{{- $bucketVersioning = .versioning }}
|
||||
{{- else }}
|
||||
{{- fail (printf "Invalid versioning value for bucket %s: %s. Must be 'Enabled' or 'Suspended'" .name .versioning) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if $bucketVersioning }}
|
||||
/bin/echo 's3.bucket.versioning -name {{ .name }} -status {{ $bucketVersioning }}' | /usr/bin/weed shell
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range $createBuckets }}
|
||||
{{- if .ttl }}
|
||||
/bin/echo 'fs.configure -locationPrefix={{ $bucketsFolder }}/{{ .name }}/ -ttl={{ .ttl }} -apply' | /usr/bin/weed shell
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range $createBuckets }}
|
||||
{{- if .anonymousRead }}
|
||||
/bin/echo \
|
||||
"s3.configure --user anonymous \
|
||||
|
||||
@@ -69,7 +69,9 @@ spec:
|
||||
priorityClassName: {{ $volume.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
{{- if $.Values.global.createClusterRole }}
|
||||
serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
|
||||
{{- end }}
|
||||
{{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
|
||||
{{- if $initContainers_exists }}
|
||||
initContainers:
|
||||
@@ -116,10 +118,8 @@ spec:
|
||||
fieldPath: status.hostIP
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" $ }}"
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" $.Values.global "component" $volume "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
{{- if $volume.extraEnvironmentVars }}
|
||||
{{- range $key, $value := $volume.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -128,6 +128,18 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if $.Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := $.Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
|
||||
@@ -93,10 +93,8 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.worker "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
{{- if .Values.worker.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.worker.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -105,6 +103,18 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
|
||||
@@ -898,15 +898,9 @@ filer:
|
||||
auditLogConfig: {}
|
||||
# You may specify buckets to be created during the install or upgrade process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
|
||||
# objectLock enables S3 Object Lock (irreversible, forces versioning)
|
||||
# versioning: Enabled or Suspended (or true to enable)
|
||||
# createBuckets:
|
||||
# - name: bucket-a
|
||||
# anonymousRead: true
|
||||
# ttl: 7d
|
||||
# objectLock: true
|
||||
# versioning: Enabled
|
||||
# - name: bucket-b
|
||||
# anonymousRead: false
|
||||
|
||||
@@ -919,13 +913,6 @@ s3:
|
||||
port: 8333
|
||||
# add additional https port
|
||||
httpsPort: 0
|
||||
# Use a custom TLS certificate secret for the S3 HTTPS endpoint.
|
||||
# When set, this Kubernetes Secret (must contain tls.crt and tls.key) is used
|
||||
# instead of the internal self-signed client certificate generated by cert-manager.
|
||||
# This allows using a publicly trusted certificate (e.g., from Let's Encrypt)
|
||||
# so that S3 clients don't need to trust the internal CA.
|
||||
# Requires global.enableSecurity to be true.
|
||||
tlsSecret: null
|
||||
metricsPort: 9327
|
||||
# Iceberg catalog REST port (Apache Iceberg REST Catalog API)
|
||||
# Set to a port number to enable, or 0/null to disable
|
||||
@@ -949,15 +936,9 @@ s3:
|
||||
auditLogConfig: {}
|
||||
# You may specify buckets to be created during the install or upgrade process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
|
||||
# objectLock enables S3 Object Lock (irreversible, forces versioning)
|
||||
# versioning: Enabled or Suspended (or true to enable)
|
||||
# createBuckets:
|
||||
# - name: bucket-a
|
||||
# anonymousRead: true
|
||||
# ttl: 7d
|
||||
# objectLock: true
|
||||
# versioning: Enabled
|
||||
# - name: bucket-b
|
||||
# anonymousRead: false
|
||||
|
||||
@@ -1460,18 +1441,11 @@ allInOne:
|
||||
# The s3-secret.yaml template only reads from .Values.s3.credentials.
|
||||
# See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
|
||||
auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig)
|
||||
trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35
|
||||
# You may specify buckets to be created during the install process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
|
||||
# objectLock enables S3 Object Lock (irreversible, forces versioning)
|
||||
# versioning: Enabled or Suspended (or true to enable)
|
||||
# createBuckets:
|
||||
# - name: bucket-a
|
||||
# anonymousRead: true
|
||||
# ttl: 7d
|
||||
# objectLock: true
|
||||
# versioning: Enabled
|
||||
# - name: bucket-b
|
||||
# anonymousRead: false
|
||||
|
||||
@@ -1637,8 +1611,6 @@ cosi:
|
||||
image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.1.2"
|
||||
driverName: "seaweedfs.objectstorage.k8s.io"
|
||||
bucketClassName: "seaweedfs"
|
||||
# Optional parameters to pass to the default BucketClass (e.g., diskType for tiered storage)
|
||||
bucketClassParameters: {}
|
||||
endpoint: ""
|
||||
region: ""
|
||||
|
||||
|
||||
@@ -100,12 +100,10 @@ message ListEntriesRequest {
|
||||
string startFromFileName = 3;
|
||||
bool inclusiveStartFrom = 4;
|
||||
uint32 limit = 5;
|
||||
int64 snapshot_ts_ns = 6;
|
||||
}
|
||||
|
||||
message ListEntriesResponse {
|
||||
Entry entry = 1;
|
||||
int64 snapshot_ts_ns = 2;
|
||||
}
|
||||
|
||||
message RemoteEntry {
|
||||
@@ -205,7 +203,6 @@ message CreateEntryRequest {
|
||||
|
||||
message CreateEntryResponse {
|
||||
string error = 1;
|
||||
SubscribeMetadataResponse metadata_event = 2;
|
||||
}
|
||||
|
||||
message UpdateEntryRequest {
|
||||
@@ -215,7 +212,6 @@ message UpdateEntryRequest {
|
||||
repeated int32 signatures = 4;
|
||||
}
|
||||
message UpdateEntryResponse {
|
||||
SubscribeMetadataResponse metadata_event = 1;
|
||||
}
|
||||
|
||||
message AppendToEntryRequest {
|
||||
@@ -240,7 +236,6 @@ message DeleteEntryRequest {
|
||||
|
||||
message DeleteEntryResponse {
|
||||
string error = 1;
|
||||
SubscribeMetadataResponse metadata_event = 2;
|
||||
}
|
||||
|
||||
message AtomicRenameEntryRequest {
|
||||
@@ -474,7 +469,6 @@ message CacheRemoteObjectToLocalClusterRequest {
|
||||
}
|
||||
message CacheRemoteObjectToLocalClusterResponse {
|
||||
Entry entry = 1;
|
||||
SubscribeMetadataResponse metadata_event = 2;
|
||||
}
|
||||
|
||||
/////////////////////////
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Multi-stage build for Go Sidecar
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git ca-certificates tzdata
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Multi-stage build for Test Client
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.23-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git ca-certificates tzdata
|
||||
|
||||
@@ -153,38 +153,6 @@ func waitForUrl(t *testing.T, url string, retries int) {
|
||||
t.Fatalf("Timeout waiting for %s", url)
|
||||
}
|
||||
|
||||
func fetchJSON(url string, out interface{}) error {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
|
||||
func mapField(obj map[string]interface{}, key string) (interface{}, bool) {
|
||||
if obj == nil {
|
||||
return nil, false
|
||||
}
|
||||
if value, ok := obj[key]; ok {
|
||||
return value, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func mapFieldAny(obj map[string]interface{}, keys ...string) (interface{}, bool) {
|
||||
for _, key := range keys {
|
||||
if value, ok := mapField(obj, key); ok {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func TestEcEndToEnd(t *testing.T) {
|
||||
defer cleanup()
|
||||
ensureEnvironment(t)
|
||||
@@ -307,7 +275,6 @@ func TestEcEndToEnd(t *testing.T) {
|
||||
startTime := time.Now()
|
||||
ecVerified := false
|
||||
var lastBody []byte
|
||||
debugTick := 0
|
||||
|
||||
for time.Since(startTime) < 300*time.Second {
|
||||
// 3.1 Check Master Topology
|
||||
@@ -333,104 +300,25 @@ func TestEcEndToEnd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 3.2 Debug: Check workers, jobs, and scheduler status
|
||||
debugTick++
|
||||
|
||||
var workers []map[string]interface{}
|
||||
// 3.2 Debug: Check workers and jobs
|
||||
wResp, wErr := http.Get(AdminUrl + "/api/plugin/workers")
|
||||
workerCount := 0
|
||||
ecDetectorCount := 0
|
||||
ecExecutorCount := 0
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/workers", &workers); err == nil {
|
||||
if wErr == nil {
|
||||
var workers []interface{}
|
||||
json.NewDecoder(wResp.Body).Decode(&workers)
|
||||
wResp.Body.Close()
|
||||
workerCount = len(workers)
|
||||
for _, worker := range workers {
|
||||
capsValue, ok := mapFieldAny(worker, "capabilities", "Capabilities")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
caps, ok := capsValue.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if capValue, ok := caps["erasure_coding"].(map[string]interface{}); ok {
|
||||
if capValue["can_detect"] == true {
|
||||
ecDetectorCount++
|
||||
}
|
||||
if capValue["can_execute"] == true {
|
||||
ecExecutorCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var tasks []map[string]interface{}
|
||||
tResp, tErr := http.Get(AdminUrl + "/api/plugin/jobs?limit=1000")
|
||||
taskCount := 0
|
||||
ecTaskCount := 0
|
||||
ecTaskStates := map[string]int{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/jobs?limit=1000", &tasks); err == nil {
|
||||
if tErr == nil {
|
||||
var tasks []interface{}
|
||||
json.NewDecoder(tResp.Body).Decode(&tasks)
|
||||
tResp.Body.Close()
|
||||
taskCount = len(tasks)
|
||||
for _, task := range tasks {
|
||||
jobType, _ := task["job_type"].(string)
|
||||
state, _ := task["state"].(string)
|
||||
if jobType == "erasure_coding" {
|
||||
ecTaskCount++
|
||||
ecTaskStates[state]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("Waiting for EC... (Workers: %d det=%d exec=%d, Tasks: %d ec=%d, EC States: %+v)",
|
||||
workerCount, ecDetectorCount, ecExecutorCount, taskCount, ecTaskCount, ecTaskStates)
|
||||
|
||||
if debugTick%3 == 0 {
|
||||
var pluginStatus map[string]interface{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/status", &pluginStatus); err == nil {
|
||||
t.Logf("Plugin status: enabled=%v worker_count=%v worker_grpc_port=%v configured=%v",
|
||||
pluginStatus["enabled"], pluginStatus["worker_count"], pluginStatus["worker_grpc_port"], pluginStatus["configured"])
|
||||
}
|
||||
|
||||
var schedulerStatus map[string]interface{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-status", &schedulerStatus); err == nil {
|
||||
if schedValue, ok := schedulerStatus["scheduler"].(map[string]interface{}); ok {
|
||||
t.Logf("Scheduler status: current_job_type=%v phase=%v last_iteration_had_jobs=%v idle_sleep_seconds=%v last_iteration_done_at=%v next_detection_at=%v",
|
||||
schedValue["current_job_type"], schedValue["current_phase"],
|
||||
schedValue["last_iteration_had_jobs"], schedValue["idle_sleep_seconds"], schedValue["last_iteration_done_at"], schedValue["next_detection_at"])
|
||||
} else {
|
||||
t.Logf("Scheduler status: %v", schedulerStatus)
|
||||
}
|
||||
}
|
||||
|
||||
var schedulerStates []map[string]interface{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-states", &schedulerStates); err == nil {
|
||||
for _, state := range schedulerStates {
|
||||
if state["job_type"] == "erasure_coding" {
|
||||
t.Logf("EC scheduler state: enabled=%v detection_in_flight=%v detector_available=%v executor_workers=%v next_detection_at=%v last_run_status=%v last_run_started_at=%v last_run_completed_at=%v",
|
||||
state["enabled"], state["detection_in_flight"], state["detector_available"],
|
||||
state["executor_worker_count"], state["next_detection_at"], state["last_run_status"],
|
||||
state["last_run_started_at"], state["last_run_completed_at"])
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var jobTypes []map[string]interface{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/job-types", &jobTypes); err == nil {
|
||||
var names []string
|
||||
for _, jobType := range jobTypes {
|
||||
if name, ok := jobType["job_type"].(string); ok && name != "" {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
t.Logf("Plugin job types: %v", names)
|
||||
}
|
||||
|
||||
var activities []map[string]interface{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/activities?job_type=erasure_coding&limit=5", &activities); err == nil {
|
||||
for i := len(activities) - 1; i >= 0; i-- {
|
||||
act := activities[i]
|
||||
t.Logf("EC activity: stage=%v message=%v occurred_at=%v", act["stage"], act["message"], act["occurred_at"])
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Logf("Waiting for EC... (Workers: %d, Active Tasks: %d)", workerCount, taskCount)
|
||||
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Simplified single-stage build for SeaweedFS with FoundationDB support
|
||||
# Force x86_64 platform to use AMD64 FoundationDB packages
|
||||
FROM --platform=linux/amd64 golang:1.25-bookworm
|
||||
FROM --platform=linux/amd64 golang:1.24-bookworm
|
||||
|
||||
ARG FOUNDATIONDB_VERSION=7.4.5
|
||||
ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Multi-stage Dockerfile to build SeaweedFS with FoundationDB support for ARM64
|
||||
FROM --platform=linux/arm64 golang:1.25-bookworm AS builder
|
||||
FROM --platform=linux/arm64 golang:1.24-bookworm AS builder
|
||||
|
||||
ARG FOUNDATIONDB_VERSION=7.4.5
|
||||
ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Test environment with Go and FoundationDB support
|
||||
FROM golang:1.25-bookworm
|
||||
FROM golang:1.24-bookworm
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
<scala.binary.version>2.12</scala.binary.version>
|
||||
<junit.version>4.13.2</junit.version>
|
||||
<seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
|
||||
<jackson.version>2.18.6</jackson.version> <!-- Upgraded from 2.15.3 -->
|
||||
<jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
|
||||
<netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
|
||||
<parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
|
||||
<parquet.format.version>2.12.0</parquet.format.version>
|
||||
@@ -123,7 +123,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<version>3.9.5</version>
|
||||
<version>3.9.4</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Apache Commons - Fix CVEs -->
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Dockerfile for Kafka Gateway Integration Testing
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git make gcc musl-dev sqlite-dev
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Dockerfile for building SeaweedFS components from the current workspace
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
RUN apk add --no-cache git make gcc musl-dev sqlite-dev
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Dockerfile for Kafka Integration Test Setup
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git make gcc musl-dev
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# Multi-stage build for cross-platform support
|
||||
|
||||
# Stage 1: Builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package ec_repair_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
func TestEcRepairDetectionFindsCandidates(t *testing.T) {
|
||||
const (
|
||||
volumeID = uint32(100)
|
||||
collection = "ec-test"
|
||||
diskType = "hdd"
|
||||
)
|
||||
|
||||
node1 := nodeSpec{
|
||||
id: "node1",
|
||||
address: "127.0.0.1:11001",
|
||||
diskType: diskType,
|
||||
diskID: 0,
|
||||
ecShards: []*master_pb.VolumeEcShardInformationMessage{
|
||||
buildEcShardInfo(volumeID, collection, diskType, 0, map[uint32]int64{
|
||||
0: 100,
|
||||
1: 100,
|
||||
2: 100,
|
||||
3: 100,
|
||||
4: 100,
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
node2 := nodeSpec{
|
||||
id: "node2",
|
||||
address: "127.0.0.1:11002",
|
||||
diskType: diskType,
|
||||
diskID: 0,
|
||||
ecShards: []*master_pb.VolumeEcShardInformationMessage{
|
||||
buildEcShardInfo(volumeID, collection, diskType, 0, map[uint32]int64{
|
||||
0: 50,
|
||||
5: 100,
|
||||
6: 100,
|
||||
7: 100,
|
||||
8: 100,
|
||||
9: 100,
|
||||
}),
|
||||
},
|
||||
}
|
||||
|
||||
topo := buildTopology([]nodeSpec{node1, node2})
|
||||
response := &master_pb.VolumeListResponse{TopologyInfo: topo}
|
||||
master := pluginworkers.NewMasterServer(t, response)
|
||||
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
handler := pluginworker.NewEcRepairHandler(dialOption)
|
||||
harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
|
||||
WorkerOptions: pluginworker.WorkerOptions{GrpcDialOption: dialOption},
|
||||
Handlers: []pluginworker.JobHandler{handler},
|
||||
})
|
||||
harness.WaitForJobType("ec_repair")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
proposals, err := harness.Plugin().RunDetection(ctx, "ec_repair", &plugin_pb.ClusterContext{
|
||||
MasterGrpcAddresses: []string{master.Address()},
|
||||
}, 10)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, proposals)
|
||||
require.Equal(t, "ec_repair", proposals[0].JobType)
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
package ec_repair_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
func TestEcRepairExecutionRepairsShards(t *testing.T) {
|
||||
const (
|
||||
volumeID = uint32(200)
|
||||
collection = "ec-repair"
|
||||
diskType = "hdd"
|
||||
)
|
||||
|
||||
volumeServers := []*pluginworkers.VolumeServer{
|
||||
pluginworkers.NewVolumeServer(t, ""),
|
||||
pluginworkers.NewVolumeServer(t, ""),
|
||||
pluginworkers.NewVolumeServer(t, ""),
|
||||
pluginworkers.NewVolumeServer(t, ""),
|
||||
}
|
||||
|
||||
nodes := []nodeSpec{
|
||||
{
|
||||
id: "node1",
|
||||
address: volumeServers[0].Address(),
|
||||
diskType: diskType,
|
||||
diskID: 0,
|
||||
ecShards: []*master_pb.VolumeEcShardInformationMessage{
|
||||
buildEcShardInfo(volumeID, collection, diskType, 0, map[uint32]int64{
|
||||
0: 100,
|
||||
1: 100,
|
||||
2: 100,
|
||||
3: 100,
|
||||
4: 100,
|
||||
}),
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "node2",
|
||||
address: volumeServers[1].Address(),
|
||||
diskType: diskType,
|
||||
diskID: 0,
|
||||
ecShards: []*master_pb.VolumeEcShardInformationMessage{
|
||||
buildEcShardInfo(volumeID, collection, diskType, 0, map[uint32]int64{
|
||||
0: 50,
|
||||
5: 100,
|
||||
6: 100,
|
||||
7: 100,
|
||||
8: 100,
|
||||
9: 100,
|
||||
}),
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "node3",
|
||||
address: volumeServers[2].Address(),
|
||||
diskType: diskType,
|
||||
diskID: 0,
|
||||
},
|
||||
{
|
||||
id: "node4",
|
||||
address: volumeServers[3].Address(),
|
||||
diskType: diskType,
|
||||
diskID: 0,
|
||||
},
|
||||
}
|
||||
|
||||
topo := buildTopology(nodes)
|
||||
response := &master_pb.VolumeListResponse{TopologyInfo: topo}
|
||||
master := pluginworkers.NewMasterServer(t, response)
|
||||
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
handler := pluginworker.NewEcRepairHandler(dialOption)
|
||||
harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
|
||||
WorkerOptions: pluginworker.WorkerOptions{GrpcDialOption: dialOption},
|
||||
Handlers: []pluginworker.JobHandler{handler},
|
||||
})
|
||||
harness.WaitForJobType("ec_repair")
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: fmt.Sprintf("ec-repair-%d", volumeID),
|
||||
JobType: "ec_repair",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"volume_id": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(volumeID)},
|
||||
},
|
||||
"collection": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: collection},
|
||||
},
|
||||
"disk_type": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: diskType},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result, err := harness.Plugin().ExecuteJob(ctx, job, &plugin_pb.ClusterContext{
|
||||
MasterGrpcAddresses: []string{master.Address()},
|
||||
}, 1)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.True(t, result.Success)
|
||||
|
||||
rebuildCalls := 0
|
||||
copyCalls := 0
|
||||
deleteCalls := 0
|
||||
unmountCalls := 0
|
||||
for _, vs := range volumeServers {
|
||||
rebuildCalls += len(vs.RebuildRequests())
|
||||
copyCalls += len(vs.CopyRequests())
|
||||
deleteCalls += len(vs.EcDeleteRequests())
|
||||
unmountCalls += len(vs.UnmountRequests())
|
||||
}
|
||||
|
||||
require.Greater(t, rebuildCalls, 0)
|
||||
require.Greater(t, copyCalls, 0)
|
||||
require.Greater(t, deleteCalls, 0)
|
||||
require.Greater(t, unmountCalls, 0)
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package ec_repair_test
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
type nodeSpec struct {
|
||||
id string
|
||||
address string
|
||||
diskType string
|
||||
diskID uint32
|
||||
ecShards []*master_pb.VolumeEcShardInformationMessage
|
||||
}
|
||||
|
||||
func buildTopology(nodes []nodeSpec) *master_pb.TopologyInfo {
|
||||
dataNodes := make([]*master_pb.DataNodeInfo, 0, len(nodes))
|
||||
for _, node := range nodes {
|
||||
diskInfo := &master_pb.DiskInfo{
|
||||
DiskId: node.diskID,
|
||||
MaxVolumeCount: 100,
|
||||
VolumeCount: 10,
|
||||
EcShardInfos: node.ecShards,
|
||||
}
|
||||
dataNodes = append(dataNodes, &master_pb.DataNodeInfo{
|
||||
Id: node.id,
|
||||
Address: node.address,
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{node.diskType: diskInfo},
|
||||
})
|
||||
}
|
||||
return &master_pb.TopologyInfo{
|
||||
DataCenterInfos: []*master_pb.DataCenterInfo{
|
||||
{
|
||||
Id: "dc1",
|
||||
RackInfos: []*master_pb.RackInfo{
|
||||
{
|
||||
Id: "rack1",
|
||||
DataNodeInfos: dataNodes,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func buildEcShardInfo(volumeID uint32, collection, diskType string, diskID uint32, shardSizes map[uint32]int64) *master_pb.VolumeEcShardInformationMessage {
|
||||
shardIDs := make([]int, 0, len(shardSizes))
|
||||
for shardID := range shardSizes {
|
||||
shardIDs = append(shardIDs, int(shardID))
|
||||
}
|
||||
sort.Ints(shardIDs)
|
||||
|
||||
var bits uint32
|
||||
sizes := make([]int64, 0, len(shardIDs))
|
||||
for _, shardID := range shardIDs {
|
||||
bits |= (1 << shardID)
|
||||
sizes = append(sizes, shardSizes[uint32(shardID)])
|
||||
}
|
||||
|
||||
return &master_pb.VolumeEcShardInformationMessage{
|
||||
Id: volumeID,
|
||||
Collection: collection,
|
||||
EcIndexBits: bits,
|
||||
DiskType: diskType,
|
||||
DiskId: diskID,
|
||||
ShardSizes: sizes,
|
||||
}
|
||||
}
|
||||
@@ -27,21 +27,23 @@ type VolumeServer struct {
|
||||
address string
|
||||
baseDir string
|
||||
|
||||
mu sync.Mutex
|
||||
receivedFiles map[string]uint64
|
||||
mountRequests []*volume_server_pb.VolumeEcShardsMountRequest
|
||||
deleteRequests []*volume_server_pb.VolumeDeleteRequest
|
||||
markReadonlyCalls int
|
||||
markWritableCalls int
|
||||
readFileStatusCalls int
|
||||
vacuumGarbageRatio float64
|
||||
vacuumCheckCalls int
|
||||
vacuumCompactCalls int
|
||||
vacuumCommitCalls int
|
||||
vacuumCleanupCalls int
|
||||
volumeCopyCalls int
|
||||
volumeMountCalls int
|
||||
tailReceiverCalls int
|
||||
mu sync.Mutex
|
||||
receivedFiles map[string]uint64
|
||||
mountRequests []*volume_server_pb.VolumeEcShardsMountRequest
|
||||
unmountRequests []*volume_server_pb.VolumeEcShardsUnmountRequest
|
||||
copyRequests []*volume_server_pb.VolumeEcShardsCopyRequest
|
||||
rebuildRequests []*volume_server_pb.VolumeEcShardsRebuildRequest
|
||||
ecDeleteRequests []*volume_server_pb.VolumeEcShardsDeleteRequest
|
||||
deleteRequests []*volume_server_pb.VolumeDeleteRequest
|
||||
markReadonlyCalls int
|
||||
vacuumGarbageRatio float64
|
||||
vacuumCheckCalls int
|
||||
vacuumCompactCalls int
|
||||
vacuumCommitCalls int
|
||||
vacuumCleanupCalls int
|
||||
volumeCopyCalls int
|
||||
volumeMountCalls int
|
||||
tailReceiverCalls int
|
||||
}
|
||||
|
||||
// NewVolumeServer starts a test volume server using the provided base directory.
|
||||
@@ -136,6 +138,46 @@ func (v *VolumeServer) MountRequests() []*volume_server_pb.VolumeEcShardsMountRe
|
||||
return out
|
||||
}
|
||||
|
||||
// UnmountRequests returns recorded unmount requests.
|
||||
func (v *VolumeServer) UnmountRequests() []*volume_server_pb.VolumeEcShardsUnmountRequest {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
|
||||
out := make([]*volume_server_pb.VolumeEcShardsUnmountRequest, len(v.unmountRequests))
|
||||
copy(out, v.unmountRequests)
|
||||
return out
|
||||
}
|
||||
|
||||
// CopyRequests returns recorded EC shard copy requests.
|
||||
func (v *VolumeServer) CopyRequests() []*volume_server_pb.VolumeEcShardsCopyRequest {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
|
||||
out := make([]*volume_server_pb.VolumeEcShardsCopyRequest, len(v.copyRequests))
|
||||
copy(out, v.copyRequests)
|
||||
return out
|
||||
}
|
||||
|
||||
// RebuildRequests returns recorded EC shard rebuild requests.
|
||||
func (v *VolumeServer) RebuildRequests() []*volume_server_pb.VolumeEcShardsRebuildRequest {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
|
||||
out := make([]*volume_server_pb.VolumeEcShardsRebuildRequest, len(v.rebuildRequests))
|
||||
copy(out, v.rebuildRequests)
|
||||
return out
|
||||
}
|
||||
|
||||
// EcDeleteRequests returns recorded EC shard delete requests.
|
||||
func (v *VolumeServer) EcDeleteRequests() []*volume_server_pb.VolumeEcShardsDeleteRequest {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
|
||||
out := make([]*volume_server_pb.VolumeEcShardsDeleteRequest, len(v.ecDeleteRequests))
|
||||
copy(out, v.ecDeleteRequests)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeleteRequests returns recorded delete requests.
|
||||
func (v *VolumeServer) DeleteRequests() []*volume_server_pb.VolumeDeleteRequest {
|
||||
v.mu.Lock()
|
||||
@@ -153,20 +195,6 @@ func (v *VolumeServer) MarkReadonlyCount() int {
|
||||
return v.markReadonlyCalls
|
||||
}
|
||||
|
||||
// MarkWritableCount returns the number of writable calls.
|
||||
func (v *VolumeServer) MarkWritableCount() int {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
return v.markWritableCalls
|
||||
}
|
||||
|
||||
// ReadFileStatusCount returns the number of ReadVolumeFileStatus calls.
|
||||
func (v *VolumeServer) ReadFileStatusCount() int {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
return v.readFileStatusCalls
|
||||
}
|
||||
|
||||
// Shutdown stops the volume server.
|
||||
func (v *VolumeServer) Shutdown() {
|
||||
if v.server != nil {
|
||||
@@ -276,6 +304,34 @@ func (v *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_serv
|
||||
return &volume_server_pb.VolumeEcShardsMountResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VolumeEcShardsUnmount(ctx context.Context, req *volume_server_pb.VolumeEcShardsUnmountRequest) (*volume_server_pb.VolumeEcShardsUnmountResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.unmountRequests = append(v.unmountRequests, req)
|
||||
v.mu.Unlock()
|
||||
return &volume_server_pb.VolumeEcShardsUnmountResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_server_pb.VolumeEcShardsCopyRequest) (*volume_server_pb.VolumeEcShardsCopyResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.copyRequests = append(v.copyRequests, req)
|
||||
v.mu.Unlock()
|
||||
return &volume_server_pb.VolumeEcShardsCopyResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_server_pb.VolumeEcShardsRebuildRequest) (*volume_server_pb.VolumeEcShardsRebuildResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.rebuildRequests = append(v.rebuildRequests, req)
|
||||
v.mu.Unlock()
|
||||
return &volume_server_pb.VolumeEcShardsRebuildResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VolumeEcShardsDelete(ctx context.Context, req *volume_server_pb.VolumeEcShardsDeleteRequest) (*volume_server_pb.VolumeEcShardsDeleteResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.ecDeleteRequests = append(v.ecDeleteRequests, req)
|
||||
v.mu.Unlock()
|
||||
return &volume_server_pb.VolumeEcShardsDeleteResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.deleteRequests = append(v.deleteRequests, req)
|
||||
@@ -296,25 +352,6 @@ func (v *VolumeServer) VolumeMarkReadonly(ctx context.Context, req *volume_serve
|
||||
return &volume_server_pb.VolumeMarkReadonlyResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VolumeMarkWritable(ctx context.Context, req *volume_server_pb.VolumeMarkWritableRequest) (*volume_server_pb.VolumeMarkWritableResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.markWritableCalls++
|
||||
v.mu.Unlock()
|
||||
return &volume_server_pb.VolumeMarkWritableResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest) (*volume_server_pb.ReadVolumeFileStatusResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.readFileStatusCalls++
|
||||
v.mu.Unlock()
|
||||
return &volume_server_pb.ReadVolumeFileStatusResponse{
|
||||
VolumeId: req.VolumeId,
|
||||
DatFileSize: 1024,
|
||||
IdxFileSize: 16,
|
||||
FileCount: 1,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VacuumVolumeCheck(ctx context.Context, req *volume_server_pb.VacuumVolumeCheckRequest) (*volume_server_pb.VacuumVolumeCheckResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.vacuumCheckCalls++
|
||||
|
||||
@@ -37,9 +37,7 @@ func TestVolumeBalanceDetectionIntegration(t *testing.T) {
|
||||
MasterGrpcAddresses: []string{master.Address()},
|
||||
}, 10)
|
||||
require.NoError(t, err)
|
||||
// With default batch_size=20 and 10 overloaded volumes vs 1 underloaded,
|
||||
// all moves are grouped into a single batch proposal.
|
||||
require.Len(t, proposals, 1, "expected exactly one batch proposal")
|
||||
require.Len(t, proposals, 1)
|
||||
|
||||
proposal := proposals[0]
|
||||
require.Equal(t, "volume_balance", proposal.JobType)
|
||||
@@ -48,15 +46,8 @@ func TestVolumeBalanceDetectionIntegration(t *testing.T) {
|
||||
|
||||
params := &worker_pb.TaskParams{}
|
||||
require.NoError(t, proto.Unmarshal(paramsValue.GetBytesValue(), params))
|
||||
|
||||
bp := params.GetBalanceParams()
|
||||
require.NotNil(t, bp, "expected BalanceParams in batch proposal")
|
||||
require.Greater(t, len(bp.Moves), 1, "batch proposal should contain multiple moves")
|
||||
for _, move := range bp.Moves {
|
||||
require.NotZero(t, move.VolumeId)
|
||||
require.NotEmpty(t, move.SourceNode)
|
||||
require.NotEmpty(t, move.TargetNode)
|
||||
}
|
||||
require.NotEmpty(t, params.Sources)
|
||||
require.NotEmpty(t, params.Targets)
|
||||
}
|
||||
|
||||
func buildBalanceVolumeListResponse(t *testing.T) *master_pb.VolumeListResponse {
|
||||
|
||||
@@ -8,12 +8,10 @@ import (
|
||||
|
||||
pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func TestVolumeBalanceExecutionIntegration(t *testing.T) {
|
||||
@@ -62,92 +60,8 @@ func TestVolumeBalanceExecutionIntegration(t *testing.T) {
|
||||
require.GreaterOrEqual(t, source.MarkReadonlyCount(), 1)
|
||||
require.GreaterOrEqual(t, len(source.DeleteRequests()), 1)
|
||||
|
||||
copyCalls, _, tailCalls := target.BalanceStats()
|
||||
copyCalls, mountCalls, tailCalls := target.BalanceStats()
|
||||
require.GreaterOrEqual(t, copyCalls, 1)
|
||||
require.GreaterOrEqual(t, mountCalls, 1)
|
||||
require.GreaterOrEqual(t, tailCalls, 1)
|
||||
}
|
||||
|
||||
func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
handler := pluginworker.NewVolumeBalanceHandler(dialOption)
|
||||
harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
|
||||
WorkerOptions: pluginworker.WorkerOptions{
|
||||
GrpcDialOption: dialOption,
|
||||
},
|
||||
Handlers: []pluginworker.JobHandler{handler},
|
||||
})
|
||||
harness.WaitForJobType("volume_balance")
|
||||
|
||||
// Create one source and one target fake volume server.
|
||||
source := pluginworkers.NewVolumeServer(t, "")
|
||||
target := pluginworkers.NewVolumeServer(t, "")
|
||||
|
||||
// Build a batch job with 3 volume moves from source → target.
|
||||
volumeIDs := []uint32{401, 402, 403}
|
||||
moves := make([]*worker_pb.BalanceMoveSpec, len(volumeIDs))
|
||||
for i, vid := range volumeIDs {
|
||||
moves[i] = &worker_pb.BalanceMoveSpec{
|
||||
VolumeId: vid,
|
||||
SourceNode: source.Address(),
|
||||
TargetNode: target.Address(),
|
||||
Collection: "batch-test",
|
||||
}
|
||||
}
|
||||
|
||||
params := &worker_pb.TaskParams{
|
||||
TaskId: "batch-balance-test",
|
||||
TaskParams: &worker_pb.TaskParams_BalanceParams{
|
||||
BalanceParams: &worker_pb.BalanceTaskParams{
|
||||
MaxConcurrentMoves: 2,
|
||||
Moves: moves,
|
||||
},
|
||||
},
|
||||
}
|
||||
paramBytes, err := proto.Marshal(params)
|
||||
require.NoError(t, err)
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: "batch-balance-test",
|
||||
JobType: "volume_balance",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"task_params_pb": {
|
||||
Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramBytes},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result, err := harness.Plugin().ExecuteJob(ctx, job, nil, 1)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.True(t, result.Success, "batch balance job should succeed; result: %+v", result)
|
||||
|
||||
// Each of the 3 moves should have marked the source readonly and deleted.
|
||||
require.Equal(t, len(volumeIDs), source.MarkReadonlyCount(),
|
||||
"each move should mark source volume readonly")
|
||||
require.Equal(t, len(volumeIDs), len(source.DeleteRequests()),
|
||||
"each move should delete the source volume")
|
||||
|
||||
// Verify delete requests reference the expected volume IDs.
|
||||
deletedVols := make(map[uint32]bool)
|
||||
for _, req := range source.DeleteRequests() {
|
||||
deletedVols[req.VolumeId] = true
|
||||
}
|
||||
for _, vid := range volumeIDs {
|
||||
require.True(t, deletedVols[vid], "volume %d should have been deleted from source", vid)
|
||||
}
|
||||
|
||||
// Pre-delete verification should have called ReadVolumeFileStatus on both
|
||||
// source and target for each volume.
|
||||
require.Equal(t, len(volumeIDs), source.ReadFileStatusCount(),
|
||||
"each move should read source volume status before delete")
|
||||
require.Equal(t, len(volumeIDs), target.ReadFileStatusCount(),
|
||||
"each move should read target volume status before delete")
|
||||
|
||||
// Target should have received copy and tail calls for all 3 volumes.
|
||||
copyCalls, _, tailCalls := target.BalanceStats()
|
||||
require.Equal(t, len(volumeIDs), copyCalls, "target should receive one copy per volume")
|
||||
require.Equal(t, len(volumeIDs), tailCalls, "target should receive one tail per volume")
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
# Install git and other build dependencies
|
||||
RUN apk add --no-cache git make
|
||||
|
||||
@@ -140,7 +140,7 @@ func TestCORSConfigurationManagement(t *testing.T) {
|
||||
Bucket: aws.String(bucketName),
|
||||
CORSConfiguration: corsConfig,
|
||||
})
|
||||
require.NoError(t, err, "Should be able to put CORS configuration")
|
||||
assert.NoError(t, err, "Should be able to put CORS configuration")
|
||||
|
||||
// Wait for metadata subscription to update cache
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
@@ -149,9 +149,9 @@ func TestCORSConfigurationManagement(t *testing.T) {
|
||||
getResp, err := client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err, "Should be able to get CORS configuration")
|
||||
require.NotNil(t, getResp.CORSRules, "CORS configuration should not be nil")
|
||||
require.Len(t, getResp.CORSRules, 1, "Should have one CORS rule")
|
||||
assert.NoError(t, err, "Should be able to get CORS configuration")
|
||||
assert.NotNil(t, getResp.CORSRules, "CORS configuration should not be nil")
|
||||
assert.Len(t, getResp.CORSRules, 1, "Should have one CORS rule")
|
||||
|
||||
rule := getResp.CORSRules[0]
|
||||
assert.Equal(t, []string{"*"}, rule.AllowedHeaders, "Allowed headers should match")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Multi-stage build for SeaweedFS S3 with IAM
|
||||
FROM golang:1.25-alpine AS builder
|
||||
FROM golang:1.23-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git make curl wget
|
||||
|
||||
@@ -185,9 +185,6 @@ test-context: ## Test only contextual policy enforcement
|
||||
test-presigned: ## Test only presigned URL integration
|
||||
go test -v -run TestS3IAMPresignedURLIntegration ./...
|
||||
|
||||
test-group: ## Run IAM group management tests
|
||||
go test -v -run "TestIAMGroup" ./...
|
||||
|
||||
test-sts: ## Run all STS tests
|
||||
go test -v -run "TestSTS" ./...
|
||||
|
||||
@@ -266,7 +263,7 @@ docker-build: ## Build custom SeaweedFS image for Docker tests
|
||||
|
||||
# All PHONY targets
|
||||
.PHONY: test test-quick run-tests setup start-services stop-services wait-for-services clean logs status debug
|
||||
.PHONY: test-auth test-policy test-expiration test-multipart test-bucket-policy test-context test-presigned test-group test-sts test-sts-assume-role test-sts-ldap
|
||||
.PHONY: test-auth test-policy test-expiration test-multipart test-bucket-policy test-context test-presigned test-sts test-sts-assume-role test-sts-ldap
|
||||
.PHONY: benchmark ci watch install-deps docker-test docker-up docker-down docker-logs docker-build
|
||||
.PHONY: test-distributed test-performance test-stress test-versioning-stress test-keycloak-full test-all-previously-skipped setup-all-tests help-advanced
|
||||
|
||||
|
||||
@@ -1,792 +0,0 @@
|
||||
package iam
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestIAMGroupLifecycle tests the full lifecycle of group management:
|
||||
// CreateGroup, GetGroup, ListGroups, DeleteGroup
|
||||
func TestIAMGroupLifecycle(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-group-lifecycle"
|
||||
|
||||
t.Run("create_group", func(t *testing.T) {
|
||||
resp, err := iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, groupName, *resp.Group.GroupName)
|
||||
})
|
||||
|
||||
t.Run("get_group", func(t *testing.T) {
|
||||
resp, err := iamClient.GetGroup(&iam.GetGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, groupName, *resp.Group.GroupName)
|
||||
})
|
||||
|
||||
t.Run("list_groups_contains_created", func(t *testing.T) {
|
||||
resp, err := iamClient.ListGroups(&iam.ListGroupsInput{})
|
||||
require.NoError(t, err)
|
||||
found := false
|
||||
for _, g := range resp.Groups {
|
||||
if *g.GroupName == groupName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Created group should appear in ListGroups")
|
||||
})
|
||||
|
||||
t.Run("create_duplicate_group_fails", func(t *testing.T) {
|
||||
_, err := iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
assert.Error(t, err, "Creating a duplicate group should fail")
|
||||
})
|
||||
|
||||
t.Run("delete_group", func(t *testing.T) {
|
||||
_, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify it's gone
|
||||
resp, err := iamClient.ListGroups(&iam.ListGroupsInput{})
|
||||
require.NoError(t, err)
|
||||
for _, g := range resp.Groups {
|
||||
assert.NotEqual(t, groupName, *g.GroupName,
|
||||
"Deleted group should not appear in ListGroups")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("delete_nonexistent_group_fails", func(t *testing.T) {
|
||||
_, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{
|
||||
GroupName: aws.String("nonexistent-group-xyz"),
|
||||
})
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupMembership tests adding and removing users from groups
|
||||
func TestIAMGroupMembership(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-group-members"
|
||||
userName := "test-user-for-group"
|
||||
|
||||
// Setup: create group and user
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
|
||||
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
|
||||
|
||||
t.Run("add_user_to_group", func(t *testing.T) {
|
||||
_, err := iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("get_group_shows_member", func(t *testing.T) {
|
||||
resp, err := iamClient.GetGroup(&iam.GetGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
found := false
|
||||
for _, u := range resp.Users {
|
||||
if *u.UserName == userName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Added user should appear in GetGroup members")
|
||||
})
|
||||
|
||||
t.Run("list_groups_for_user", func(t *testing.T) {
|
||||
resp, err := iamClient.ListGroupsForUser(&iam.ListGroupsForUserInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
found := false
|
||||
for _, g := range resp.Groups {
|
||||
if *g.GroupName == groupName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Group should appear in ListGroupsForUser")
|
||||
})
|
||||
|
||||
t.Run("add_duplicate_member_is_idempotent", func(t *testing.T) {
|
||||
_, err := iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
// Should succeed (idempotent) or return a benign error
|
||||
// AWS IAM allows duplicate add without error
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("remove_user_from_group", func(t *testing.T) {
|
||||
_, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify removal
|
||||
resp, err := iamClient.GetGroup(&iam.GetGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
for _, u := range resp.Users {
|
||||
assert.NotEqual(t, userName, *u.UserName,
|
||||
"Removed user should not appear in group members")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupPolicyAttachment tests attaching and detaching policies from groups
|
||||
func TestIAMGroupPolicyAttachment(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-group-policies"
|
||||
policyName := "test-group-attach-policy"
|
||||
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"*"}]}`
|
||||
|
||||
// Setup: create group and policy
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
policyArn := createPolicyResp.Policy.Arn
|
||||
|
||||
// Cleanup in correct order: detach policy, delete group, delete policy
|
||||
t.Cleanup(func() {
|
||||
if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: policyArn,
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to detach group policy: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete group: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn}); err != nil {
|
||||
t.Logf("cleanup: failed to delete policy: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("attach_group_policy", func(t *testing.T) {
|
||||
_, err := iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: policyArn,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("list_attached_group_policies", func(t *testing.T) {
|
||||
resp, err := iamClient.ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
found := false
|
||||
for _, p := range resp.AttachedPolicies {
|
||||
if *p.PolicyName == policyName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Attached policy should appear in ListAttachedGroupPolicies")
|
||||
})
|
||||
|
||||
t.Run("detach_group_policy", func(t *testing.T) {
|
||||
_, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: policyArn,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify detachment
|
||||
resp, err := iamClient.ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
for _, p := range resp.AttachedPolicies {
|
||||
assert.NotEqual(t, policyName, *p.PolicyName,
|
||||
"Detached policy should not appear in ListAttachedGroupPolicies")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupPolicyEnforcement tests that group policies are enforced during S3 operations.
|
||||
// Creates a user with no direct policies, adds them to a group with S3 access,
|
||||
// and verifies they can access S3 through the group policy.
|
||||
func TestIAMGroupPolicyEnforcement(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-enforcement-group"
|
||||
userName := "test-enforcement-user"
|
||||
policyName := "test-enforcement-policy"
|
||||
bucketName := "test-group-enforce-bucket"
|
||||
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"]}]}`
|
||||
|
||||
// Create user
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create access key for the user
|
||||
keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
accessKeyId := *keyResp.AccessKey.AccessKeyId
|
||||
secretKey := *keyResp.AccessKey.SecretAccessKey
|
||||
|
||||
// Create an S3 client with the user's credentials
|
||||
userS3Client := createS3Client(t, accessKeyId, secretKey)
|
||||
|
||||
// Create group
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create policy
|
||||
createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
policyArn := createPolicyResp.Policy.Arn
|
||||
|
||||
// Cleanup in correct order: remove user from group, detach policy,
|
||||
// delete access key, delete user, delete group, delete policy
|
||||
t.Cleanup(func() {
|
||||
if _, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to remove user from group: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: policyArn,
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to detach group policy: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
|
||||
UserName: aws.String(userName),
|
||||
AccessKeyId: keyResp.AccessKey.AccessKeyId,
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to delete access key: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete user: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete group: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn}); err != nil {
|
||||
t.Logf("cleanup: failed to delete policy: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Register bucket cleanup on parent test with admin credentials
|
||||
// (userS3Client may lack permissions by cleanup time)
|
||||
adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
if _, err := adminS3.DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("test-key"),
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to delete object: %v", err)
|
||||
}
|
||||
if _, err := adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete bucket: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("user_without_group_denied", func(t *testing.T) {
|
||||
// User has no policies and is not in any group — should be denied
|
||||
_, err := userS3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err, "User without any policies should be denied")
|
||||
awsErr, ok := err.(awserr.Error)
|
||||
require.True(t, ok, "Expected awserr.Error")
|
||||
assert.Equal(t, "AccessDenied", awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("user_with_group_policy_allowed", func(t *testing.T) {
|
||||
// Attach policy to group
|
||||
_, err := iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: policyArn,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add user to group
|
||||
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait for policy propagation, then create bucket
|
||||
require.Eventually(t, func() bool {
|
||||
_, err = userS3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return err == nil
|
||||
}, 10*time.Second, 500*time.Millisecond, "User with group policy should be allowed")
|
||||
|
||||
// Should also be able to put/get objects
|
||||
_, err = userS3Client.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("test-key"),
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("test-data")),
|
||||
})
|
||||
require.NoError(t, err, "User should be able to put objects through group policy")
|
||||
})
|
||||
|
||||
t.Run("user_removed_from_group_denied", func(t *testing.T) {
|
||||
// Remove user from group
|
||||
_, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait for policy propagation — user should now be denied
|
||||
var lastErr error
|
||||
require.Eventually(t, func() bool {
|
||||
_, lastErr = userS3Client.ListObjects(&s3.ListObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return lastErr != nil
|
||||
}, 10*time.Second, 500*time.Millisecond, "User removed from group should be denied")
|
||||
awsErr, ok := lastErr.(awserr.Error)
|
||||
require.True(t, ok, "Expected awserr.Error")
|
||||
assert.Equal(t, "AccessDenied", awsErr.Code())
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupDisabledPolicyEnforcement tests that disabled groups do not contribute policies.
|
||||
// Uses the raw IAM API (callIAMAPI) since the AWS SDK doesn't support custom group status.
|
||||
func TestIAMGroupDisabledPolicyEnforcement(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
if !isSeaweedFSRunning(t) {
|
||||
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
|
||||
}
|
||||
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-disabled-group"
|
||||
userName := "test-disabled-grp-user"
|
||||
policyName := "test-disabled-grp-policy"
|
||||
bucketName := "test-disabled-grp-bucket"
|
||||
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"]}]}`
|
||||
|
||||
// Create user, group, policy
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(groupName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName), PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Cleanup in correct order: remove user from group, detach policy,
|
||||
// delete access key, delete user, delete group, delete policy
|
||||
t.Cleanup(func() {
|
||||
if _, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(groupName), UserName: aws.String(userName),
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to remove user from group: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: aws.String("arn:aws:iam:::policy/" + policyName),
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to detach group policy: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
|
||||
UserName: aws.String(userName), AccessKeyId: keyResp.AccessKey.AccessKeyId,
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to delete access key: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete user: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete group: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: createPolicyResp.Policy.Arn}); err != nil {
|
||||
t.Logf("cleanup: failed to delete policy: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Setup: attach policy, add user, create bucket with admin
|
||||
_, err = iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName), PolicyArn: createPolicyResp.Policy.Arn,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(groupName), UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
userS3Client := createS3Client(t, *keyResp.AccessKey.AccessKeyId, *keyResp.AccessKey.SecretAccessKey)
|
||||
|
||||
// Create bucket using admin first so we can test listing
|
||||
adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
_, err = adminS3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucketName)})
|
||||
require.NoError(t, err)
|
||||
defer adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
|
||||
|
||||
t.Run("enabled_group_allows_access", func(t *testing.T) {
|
||||
require.Eventually(t, func() bool {
|
||||
_, err := userS3Client.ListObjects(&s3.ListObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return err == nil
|
||||
}, 10*time.Second, 500*time.Millisecond, "User in enabled group should have access")
|
||||
})
|
||||
|
||||
t.Run("disabled_group_denies_access", func(t *testing.T) {
|
||||
// Disable group via raw IAM API (no SDK support for this extension)
|
||||
resp, err := callIAMAPIAuthenticated(t, framework, "UpdateGroup", url.Values{
|
||||
"GroupName": {groupName},
|
||||
"Disabled": {"true"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "UpdateGroup (disable) should return 200")
|
||||
|
||||
// Wait for propagation — user should be denied
|
||||
var lastErr error
|
||||
require.Eventually(t, func() bool {
|
||||
_, lastErr = userS3Client.ListObjects(&s3.ListObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return lastErr != nil
|
||||
}, 10*time.Second, 500*time.Millisecond, "User in disabled group should be denied access")
|
||||
awsErr, ok := lastErr.(awserr.Error)
|
||||
require.True(t, ok, "Expected awserr.Error")
|
||||
assert.Equal(t, "AccessDenied", awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("re_enabled_group_restores_access", func(t *testing.T) {
|
||||
// Re-enable the group
|
||||
resp, err := callIAMAPIAuthenticated(t, framework, "UpdateGroup", url.Values{
|
||||
"GroupName": {groupName},
|
||||
"Disabled": {"false"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "UpdateGroup (re-enable) should return 200")
|
||||
|
||||
// Wait for propagation — user should have access again
|
||||
require.Eventually(t, func() bool {
|
||||
_, err = userS3Client.ListObjects(&s3.ListObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return err == nil
|
||||
}, 10*time.Second, 500*time.Millisecond, "User in re-enabled group should have access again")
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupUserDeletionSideEffect tests that deleting a user removes them from all groups.
|
||||
func TestIAMGroupUserDeletionSideEffect(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-deletion-group"
|
||||
userName := "test-deletion-user"
|
||||
|
||||
// Create group and user
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(groupName)})
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
|
||||
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
// Best-effort: user may already be deleted by the test
|
||||
iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
|
||||
})
|
||||
|
||||
// Add user to group
|
||||
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify user is in group
|
||||
getResp, err := iamClient.GetGroup(&iam.GetGroupInput{GroupName: aws.String(groupName)})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, getResp.Users, 1, "Group should have 1 member before deletion")
|
||||
|
||||
// Delete the user
|
||||
_, err = iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify user was removed from the group
|
||||
getResp, err = iamClient.GetGroup(&iam.GetGroupInput{GroupName: aws.String(groupName)})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, getResp.Users, "Group should have no members after user deletion")
|
||||
}
|
||||
|
||||
// TestIAMGroupMultipleGroups tests that a user can belong to multiple groups
|
||||
// and inherits policies from all of them.
|
||||
func TestIAMGroupMultipleGroups(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
group1 := "test-multi-group-1"
|
||||
group2 := "test-multi-group-2"
|
||||
userName := "test-multi-group-user"
|
||||
|
||||
// Create two groups
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(group1)})
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(group1)})
|
||||
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(group2)})
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(group2)})
|
||||
|
||||
// Create user
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(group1), UserName: aws.String(userName),
|
||||
})
|
||||
iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(group2), UserName: aws.String(userName),
|
||||
})
|
||||
iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
|
||||
}()
|
||||
|
||||
// Add user to both groups
|
||||
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(group1), UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(group2), UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify user appears in both groups
|
||||
resp, err := iamClient.ListGroupsForUser(&iam.ListGroupsForUserInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
groupNames := make(map[string]bool)
|
||||
for _, g := range resp.Groups {
|
||||
groupNames[*g.GroupName] = true
|
||||
}
|
||||
assert.True(t, groupNames[group1], "User should be in group 1")
|
||||
assert.True(t, groupNames[group2], "User should be in group 2")
|
||||
}
|
||||
|
||||
// --- Response types for raw IAM API calls ---
|
||||
|
||||
type CreateGroupResponse struct {
|
||||
XMLName xml.Name `xml:"CreateGroupResponse"`
|
||||
CreateGroupResult struct {
|
||||
Group struct {
|
||||
GroupName string `xml:"GroupName"`
|
||||
} `xml:"Group"`
|
||||
} `xml:"CreateGroupResult"`
|
||||
}
|
||||
|
||||
type ListGroupsResponse struct {
|
||||
XMLName xml.Name `xml:"ListGroupsResponse"`
|
||||
ListGroupsResult struct {
|
||||
Groups []struct {
|
||||
GroupName string `xml:"GroupName"`
|
||||
} `xml:"Groups>member"`
|
||||
} `xml:"ListGroupsResult"`
|
||||
}
|
||||
|
||||
// callIAMAPIAuthenticated sends an authenticated raw IAM API request using the
|
||||
// framework's JWT token. This is needed for custom extensions not in the AWS SDK
|
||||
// (like UpdateGroup with Disabled parameter).
|
||||
func callIAMAPIAuthenticated(_ *testing.T, framework *S3IAMTestFramework, action string, params url.Values) (*http.Response, error) {
|
||||
params.Set("Action", action)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, TestIAMEndpoint+"/",
|
||||
strings.NewReader(params.Encode()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
token, err := framework.generateSTSSessionToken("admin-user", "TestAdminRole", time.Hour, "", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &BearerTokenTransport{Token: token},
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
// TestIAMGroupRawAPI tests group operations using raw HTTP IAM API calls,
|
||||
// verifying XML response format for group operations.
|
||||
func TestIAMGroupRawAPI(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
if !isSeaweedFSRunning(t) {
|
||||
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
|
||||
}
|
||||
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
groupName := "test-raw-api-group"
|
||||
|
||||
t.Run("create_group_raw", func(t *testing.T) {
|
||||
resp, err := callIAMAPIAuthenticated(t, framework, "CreateGroup", url.Values{
|
||||
"GroupName": {groupName},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
|
||||
var createResp CreateGroupResponse
|
||||
err = xml.Unmarshal(body, &createResp)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, groupName, createResp.CreateGroupResult.Group.GroupName)
|
||||
})
|
||||
|
||||
t.Run("list_groups_raw", func(t *testing.T) {
|
||||
resp, err := callIAMAPIAuthenticated(t, framework, "ListGroups", url.Values{})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
|
||||
var listResp ListGroupsResponse
|
||||
err = xml.Unmarshal(body, &listResp)
|
||||
require.NoError(t, err)
|
||||
|
||||
found := false
|
||||
for _, g := range listResp.ListGroupsResult.Groups {
|
||||
if g.GroupName == groupName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Created group should appear in raw ListGroups")
|
||||
})
|
||||
|
||||
t.Run("delete_group_raw", func(t *testing.T) {
|
||||
resp, err := callIAMAPIAuthenticated(t, framework, "DeleteGroup", url.Values{
|
||||
"GroupName": {groupName},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
})
|
||||
}
|
||||
|
||||
// createS3Client creates an S3 client with static credentials
|
||||
func createS3Client(t *testing.T, accessKey, secretKey string) *s3.S3 {
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String("us-east-1"),
|
||||
Endpoint: aws.String(TestS3Endpoint),
|
||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return s3.New(sess)
|
||||
}
|
||||
@@ -1,573 +0,0 @@
|
||||
package example
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
v1credentials "github.com/aws/aws-sdk-go/aws/credentials"
|
||||
v1signer "github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||
v1s3 "github.com/aws/aws-sdk-go/service/s3"
|
||||
v2aws "github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
v2s3 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newS3V2Client creates an AWS SDK v2 S3 client from the test cluster.
|
||||
func newS3V2Client(cluster *TestCluster) *v2s3.Client {
|
||||
return v2s3.New(v2s3.Options{
|
||||
Region: testRegion,
|
||||
BaseEndpoint: v2aws.String(cluster.s3Endpoint),
|
||||
Credentials: v2aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, "")),
|
||||
UsePathStyle: true,
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetObjectAttributes(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
t.Run("Basic", func(t *testing.T) {
|
||||
testGetObjectAttributesBasic(t, cluster)
|
||||
})
|
||||
t.Run("MultipartObject", func(t *testing.T) {
|
||||
testGetObjectAttributesMultipart(t, cluster)
|
||||
})
|
||||
t.Run("SelectiveAttributes", func(t *testing.T) {
|
||||
testGetObjectAttributesSelective(t, cluster)
|
||||
})
|
||||
t.Run("InvalidAttribute", func(t *testing.T) {
|
||||
testGetObjectAttributesInvalid(t, cluster)
|
||||
})
|
||||
t.Run("NonExistentObject", func(t *testing.T) {
|
||||
testGetObjectAttributesNotFound(t, cluster)
|
||||
})
|
||||
t.Run("VersionedObject", func(t *testing.T) {
|
||||
testGetObjectAttributesVersioned(t, cluster)
|
||||
})
|
||||
t.Run("ConditionalHeaders", func(t *testing.T) {
|
||||
testGetObjectAttributesConditionalHeaders(t, cluster)
|
||||
})
|
||||
t.Run("VersionedConditionalHeaders", func(t *testing.T) {
|
||||
testGetObjectAttributesVersionedConditionalHeaders(t, cluster)
|
||||
})
|
||||
}
|
||||
|
||||
func testGetObjectAttributesBasic(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-basic-")
|
||||
objectKey := "test-object.txt"
|
||||
objectData := "Hello, GetObjectAttributes!"
|
||||
|
||||
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte(objectData)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
client := newS3V2Client(cluster)
|
||||
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String(objectKey),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesEtag,
|
||||
types.ObjectAttributesStorageClass,
|
||||
types.ObjectAttributesObjectSize,
|
||||
types.ObjectAttributesObjectParts,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// ETag should be present and non-empty
|
||||
require.NotNil(t, resp.ETag)
|
||||
assert.NotEmpty(t, *resp.ETag)
|
||||
assert.False(t, strings.Contains(*resp.ETag, `"`), "ETag in XML body should not have quotes")
|
||||
|
||||
// ObjectSize should match
|
||||
require.NotNil(t, resp.ObjectSize)
|
||||
assert.Equal(t, int64(len(objectData)), *resp.ObjectSize)
|
||||
|
||||
// StorageClass should be STANDARD (default)
|
||||
assert.Equal(t, "STANDARD", string(resp.StorageClass))
|
||||
|
||||
// ObjectParts should be nil for non-multipart objects
|
||||
assert.Nil(t, resp.ObjectParts)
|
||||
|
||||
// LastModified header should be present
|
||||
assert.NotNil(t, resp.LastModified)
|
||||
|
||||
t.Logf("Basic GetObjectAttributes passed: ETag=%s, Size=%d, StorageClass=%s",
|
||||
*resp.ETag, *resp.ObjectSize, resp.StorageClass)
|
||||
}
|
||||
|
||||
func testGetObjectAttributesMultipart(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-mp-")
|
||||
objectKey := "test-multipart.bin"
|
||||
|
||||
// Create a 2-part multipart upload
|
||||
part1Data := bytes.Repeat([]byte("A"), 5*1024*1024) // 5MB (minimum part size)
|
||||
part2Data := bytes.Repeat([]byte("B"), 3*1024*1024) // 3MB
|
||||
|
||||
initResp, err := cluster.s3Client.CreateMultipartUpload(&v1s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
uploadID := initResp.UploadId
|
||||
|
||||
part1Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
PartNumber: aws.Int64(1),
|
||||
UploadId: uploadID,
|
||||
Body: bytes.NewReader(part1Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
part2Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
PartNumber: aws.Int64(2),
|
||||
UploadId: uploadID,
|
||||
Body: bytes.NewReader(part2Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = cluster.s3Client.CompleteMultipartUpload(&v1s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadID,
|
||||
MultipartUpload: &v1s3.CompletedMultipartUpload{
|
||||
Parts: []*v1s3.CompletedPart{
|
||||
{ETag: part1Resp.ETag, PartNumber: aws.Int64(1)},
|
||||
{ETag: part2Resp.ETag, PartNumber: aws.Int64(2)},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait briefly for metadata to settle
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
client := newS3V2Client(cluster)
|
||||
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String(objectKey),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesObjectParts,
|
||||
types.ObjectAttributesObjectSize,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NotNil(t, resp.ObjectSize)
|
||||
assert.Equal(t, int64(len(part1Data)+len(part2Data)), *resp.ObjectSize)
|
||||
|
||||
require.NotNil(t, resp.ObjectParts, "ObjectParts should be present for multipart objects")
|
||||
assert.Equal(t, int32(2), *resp.ObjectParts.TotalPartsCount)
|
||||
require.Len(t, resp.ObjectParts.Parts, 2)
|
||||
assert.Equal(t, int32(1), *resp.ObjectParts.Parts[0].PartNumber)
|
||||
assert.Equal(t, int64(len(part1Data)), *resp.ObjectParts.Parts[0].Size)
|
||||
assert.Equal(t, int32(2), *resp.ObjectParts.Parts[1].PartNumber)
|
||||
assert.Equal(t, int64(len(part2Data)), *resp.ObjectParts.Parts[1].Size)
|
||||
|
||||
// Test pagination: MaxParts=1
|
||||
resp2, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String(objectKey),
|
||||
MaxParts: v2aws.Int32(1),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesObjectParts,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp2.ObjectParts)
|
||||
assert.Len(t, resp2.ObjectParts.Parts, 1)
|
||||
assert.True(t, *resp2.ObjectParts.IsTruncated)
|
||||
assert.Equal(t, int32(2), *resp2.ObjectParts.TotalPartsCount)
|
||||
|
||||
t.Logf("Multipart GetObjectAttributes passed: %d parts, total size %d",
|
||||
*resp.ObjectParts.TotalPartsCount, *resp.ObjectSize)
|
||||
}
|
||||
|
||||
func testGetObjectAttributesSelective(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-sel-")
|
||||
objectKey := "test-selective.txt"
|
||||
objectData := "Selective attributes test"
|
||||
|
||||
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte(objectData)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
client := newS3V2Client(cluster)
|
||||
|
||||
// Request only ETag
|
||||
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String(objectKey),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesEtag,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp.ETag)
|
||||
assert.NotEmpty(t, *resp.ETag)
|
||||
assert.Nil(t, resp.ObjectSize, "ObjectSize should not be present when not requested")
|
||||
assert.Empty(t, string(resp.StorageClass), "StorageClass should not be present when not requested")
|
||||
assert.Nil(t, resp.ObjectParts, "ObjectParts should not be present when not requested")
|
||||
|
||||
t.Logf("Selective GetObjectAttributes passed: ETag=%s", *resp.ETag)
|
||||
}
|
||||
|
||||
func testGetObjectAttributesInvalid(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-inv-")
|
||||
objectKey := "test-object.txt"
|
||||
|
||||
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte("test")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Use raw HTTP to send an invalid attribute name since the SDK validates
|
||||
reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Object-Attributes", "InvalidAttr")
|
||||
|
||||
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
|
||||
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
|
||||
require.NoError(t, err)
|
||||
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
|
||||
assert.Equal(t, 400, resp.StatusCode)
|
||||
t.Logf("Invalid attribute test passed: got %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
func testGetObjectAttributesNotFound(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-nf-")
|
||||
|
||||
client := newS3V2Client(cluster)
|
||||
_, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("nonexistent-key"),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesEtag,
|
||||
},
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "NoSuchKey")
|
||||
|
||||
t.Logf("NotFound GetObjectAttributes passed")
|
||||
}
|
||||
|
||||
func testGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster) {
|
||||
client := newS3V2Client(cluster)
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-ver-")
|
||||
|
||||
// Enable versioning
|
||||
_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusEnabled,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
// Put two versions of the same object
|
||||
v1Data := "version 1 content"
|
||||
putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("versioned-key"),
|
||||
Body: strings.NewReader(v1Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp1.VersionId)
|
||||
versionId1 := *putResp1.VersionId
|
||||
|
||||
v2Data := "version 2 content - longer"
|
||||
putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("versioned-key"),
|
||||
Body: strings.NewReader(v2Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp2.VersionId)
|
||||
versionId2 := *putResp2.VersionId
|
||||
|
||||
assert.NotEqual(t, versionId1, versionId2, "versions should differ")
|
||||
|
||||
// GetObjectAttributes for latest version (v2)
|
||||
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("versioned-key"),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesObjectSize,
|
||||
types.ObjectAttributesEtag,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp.ObjectSize)
|
||||
assert.Equal(t, int64(len(v2Data)), *resp.ObjectSize)
|
||||
require.NotNil(t, resp.VersionId)
|
||||
assert.Equal(t, versionId2, *resp.VersionId)
|
||||
|
||||
// GetObjectAttributes for specific older version (v1)
|
||||
resp1, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("versioned-key"),
|
||||
VersionId: v2aws.String(versionId1),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesObjectSize,
|
||||
types.ObjectAttributesEtag,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp1.ObjectSize)
|
||||
assert.Equal(t, int64(len(v1Data)), *resp1.ObjectSize)
|
||||
require.NotNil(t, resp1.VersionId)
|
||||
assert.Equal(t, versionId1, *resp1.VersionId)
|
||||
|
||||
t.Logf("Versioned GetObjectAttributes passed: v1 size=%d (id=%s), v2 size=%d (id=%s)",
|
||||
*resp1.ObjectSize, versionId1, *resp.ObjectSize, versionId2)
|
||||
}
|
||||
|
||||
// signedGetObjectAttributes creates a signed GET request for ?attributes with custom headers.
|
||||
func signedGetObjectAttributes(t *testing.T, cluster *TestCluster, bucketName, objectKey string, extraHeaders map[string]string) *http.Response {
|
||||
reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
|
||||
for k, v := range extraHeaders {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
|
||||
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
|
||||
require.NoError(t, err)
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
return resp
|
||||
}
|
||||
|
||||
func testGetObjectAttributesConditionalHeaders(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-cond-")
|
||||
objectKey := "cond-test.txt"
|
||||
|
||||
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte("conditional headers test")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get the ETag and Last-Modified for the object
|
||||
headResp, err := cluster.s3Client.HeadObject(&v1s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
etag := aws.StringValue(headResp.ETag)
|
||||
lastModified := headResp.LastModified
|
||||
require.NotNil(t, lastModified)
|
||||
|
||||
pastDate := lastModified.Add(-1 * time.Hour).UTC().Format(http.TimeFormat)
|
||||
futureDate := lastModified.Add(1 * time.Hour).UTC().Format(http.TimeFormat)
|
||||
|
||||
// RFC 7232: If-Match true + If-Unmodified-Since false => 200 OK
|
||||
// If-Unmodified-Since is ignored when If-Match is present
|
||||
t.Run("IfMatch_true_IfUnmodifiedSince_false", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
|
||||
"If-Match": etag,
|
||||
"If-Unmodified-Since": pastDate, // object was modified after this => false
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 200, resp.StatusCode,
|
||||
"If-Match=true should return 200 even when If-Unmodified-Since=false (RFC 7232 Section 3.4)")
|
||||
})
|
||||
|
||||
// RFC 7232: If-None-Match false + If-Modified-Since true => 304 Not Modified
|
||||
// If-Modified-Since is ignored when If-None-Match is present
|
||||
t.Run("IfNoneMatch_false_IfModifiedSince_true", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
|
||||
"If-None-Match": etag,
|
||||
"If-Modified-Since": pastDate, // object was modified after this => true
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 304, resp.StatusCode,
|
||||
"If-None-Match=false (ETag match) should return 304 even when If-Modified-Since=true (RFC 7232 Section 3.3)")
|
||||
})
|
||||
|
||||
// If-Match succeeds, If-Unmodified-Since also succeeds => 200
|
||||
t.Run("IfMatch_true_IfUnmodifiedSince_true", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
|
||||
"If-Match": etag,
|
||||
"If-Unmodified-Since": futureDate,
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 200, resp.StatusCode)
|
||||
})
|
||||
|
||||
// If-None-Match passes (ETag differs), If-Modified-Since ignored => 200
|
||||
// Per RFC 7232, If-Modified-Since is ignored when If-None-Match is present
|
||||
t.Run("IfNoneMatch_true_IfModifiedSince_ignored", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
|
||||
"If-None-Match": `"nonexistent-etag"`,
|
||||
"If-Modified-Since": futureDate, // would fail alone, but is ignored
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 200, resp.StatusCode,
|
||||
"If-None-Match=true means If-Modified-Since is ignored, should return 200 (RFC 7232 Section 3.3)")
|
||||
})
|
||||
|
||||
// If-Match fails => 412 regardless of If-Unmodified-Since
|
||||
t.Run("IfMatch_false", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
|
||||
"If-Match": `"wrong-etag"`,
|
||||
"If-Unmodified-Since": futureDate,
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 412, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Logf("Conditional headers tests passed")
|
||||
}
|
||||
|
||||
// signedGetObjectAttributesVersioned creates a signed GET request for ?attributes&versionId=... with custom headers.
|
||||
func signedGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster, bucketName, objectKey, versionId string, extraHeaders map[string]string) *http.Response {
|
||||
reqURL := fmt.Sprintf("%s/%s/%s?attributes&versionId=%s", cluster.s3Endpoint, bucketName, objectKey, versionId)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
|
||||
for k, v := range extraHeaders {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
|
||||
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
|
||||
require.NoError(t, err)
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
return resp
|
||||
}
|
||||
|
||||
func testGetObjectAttributesVersionedConditionalHeaders(t *testing.T, cluster *TestCluster) {
|
||||
client := newS3V2Client(cluster)
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-vcond-")
|
||||
|
||||
// Enable versioning
|
||||
_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusEnabled,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
// Put two versions with different content (different ETags)
|
||||
v1Data := "version 1 - original"
|
||||
putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("vcond-key"),
|
||||
Body: strings.NewReader(v1Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp1.VersionId)
|
||||
vid1 := *putResp1.VersionId
|
||||
|
||||
v2Data := "version 2 - updated content"
|
||||
putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("vcond-key"),
|
||||
Body: strings.NewReader(v2Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp2.VersionId)
|
||||
vid2 := *putResp2.VersionId
|
||||
|
||||
// Get ETags for each version
|
||||
headV1, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("vcond-key"),
|
||||
VersionId: v2aws.String(vid1),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
etagV1 := *headV1.ETag
|
||||
|
||||
headV2, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("vcond-key"),
|
||||
VersionId: v2aws.String(vid2),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
etagV2 := *headV2.ETag
|
||||
require.NotEqual(t, etagV1, etagV2, "versions should have different ETags")
|
||||
|
||||
// If-Match with v1's ETag + versionId=v1 => 200
|
||||
// Before the fix, this would fail with 412 because conditional headers
|
||||
// were evaluated against the latest version (v2) whose ETag differs
|
||||
t.Run("IfMatch_v1_etag_versionId_v1", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
|
||||
"If-Match": etagV1,
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 200, resp.StatusCode,
|
||||
"If-Match with v1 ETag targeting versionId=v1 should return 200")
|
||||
})
|
||||
|
||||
// If-Match with v2's ETag + versionId=v1 => 412
|
||||
// The ETag doesn't match v1, so this should fail
|
||||
t.Run("IfMatch_v2_etag_versionId_v1", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
|
||||
"If-Match": etagV2,
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 412, resp.StatusCode,
|
||||
"If-Match with v2 ETag targeting versionId=v1 should return 412")
|
||||
})
|
||||
|
||||
// If-None-Match with v1's ETag + versionId=v1 => 304
|
||||
t.Run("IfNoneMatch_v1_etag_versionId_v1", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
|
||||
"If-None-Match": etagV1,
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 304, resp.StatusCode,
|
||||
"If-None-Match with v1 ETag targeting versionId=v1 should return 304")
|
||||
})
|
||||
|
||||
t.Logf("Versioned conditional headers tests passed: vid1=%s, vid2=%s", vid1, vid2)
|
||||
}
|
||||
@@ -422,218 +422,6 @@ func TestS3MultipartOperationsInheritPutObjectPermissions(t *testing.T) {
|
||||
require.Equal(t, 0, len(listUploadsOut.Uploads))
|
||||
}
|
||||
|
||||
// TestS3IAMManagedPolicyLifecycle is an end-to-end integration test covering the
|
||||
// user-reported use case in https://github.com/seaweedfs/seaweedfs/issues/8506
|
||||
// where managed policy operations (GetPolicy, ListPolicies, DeletePolicy,
|
||||
// AttachUserPolicy, DetachUserPolicy) returned 500 errors.
|
||||
func TestS3IAMManagedPolicyLifecycle(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
iamClient := newIAMClient(t, cluster.s3Endpoint)
|
||||
|
||||
// Step 1: Create a user (this already worked per the issue)
|
||||
userName := uniqueName("lifecycle-user")
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err, "CreateUser should succeed")
|
||||
|
||||
// Step 2: Create a managed policy via IAM API
|
||||
policyName := uniqueName("lifecycle-policy")
|
||||
policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
|
||||
policyDoc := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject"],
|
||||
"Resource": "arn:aws:s3:::*"
|
||||
}]
|
||||
}`
|
||||
createOut, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err, "CreatePolicy should succeed")
|
||||
require.NotNil(t, createOut.Policy)
|
||||
require.Equal(t, policyName, *createOut.Policy.PolicyName)
|
||||
|
||||
// Step 3: ListPolicies — should include the created policy (was returning 500)
|
||||
listOut, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
|
||||
require.NoError(t, err, "ListPolicies should succeed (was returning 500)")
|
||||
require.True(t, managedPolicyContains(listOut.Policies, policyName),
|
||||
"ListPolicies should contain the newly created policy")
|
||||
|
||||
// Step 4: GetPolicy by ARN — should return the policy (was returning 500)
|
||||
getOut, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
|
||||
require.NoError(t, err, "GetPolicy should succeed (was returning 500)")
|
||||
require.NotNil(t, getOut.Policy)
|
||||
require.Equal(t, policyName, *getOut.Policy.PolicyName)
|
||||
require.Equal(t, policyArn, *getOut.Policy.Arn)
|
||||
|
||||
// Step 5: AttachUserPolicy — should succeed (was returning 500)
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(policyArn),
|
||||
})
|
||||
require.NoError(t, err, "AttachUserPolicy should succeed (was returning 500)")
|
||||
|
||||
// Step 6: ListAttachedUserPolicies — verify the policy is attached
|
||||
attachedOut, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err, "ListAttachedUserPolicies should succeed")
|
||||
require.True(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
|
||||
"Policy should appear in user's attached policies")
|
||||
|
||||
// Step 7: Idempotent re-attach should not fail
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(policyArn),
|
||||
})
|
||||
require.NoError(t, err, "Re-attaching same policy should be idempotent")
|
||||
|
||||
// Step 8: DeletePolicy while attached — should fail with DeleteConflict (AWS behavior)
|
||||
_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
|
||||
require.Error(t, err, "DeletePolicy should fail while policy is attached")
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeDeleteConflictException, awsErr.Code(),
|
||||
"Should return DeleteConflict when deleting attached policy")
|
||||
|
||||
// Step 9: DetachUserPolicy
|
||||
_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(policyArn),
|
||||
})
|
||||
require.NoError(t, err, "DetachUserPolicy should succeed")
|
||||
|
||||
// Verify detached
|
||||
attachedOut, err = iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.False(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
|
||||
"Policy should no longer appear in user's attached policies after detach")
|
||||
|
||||
// Step 10: DeletePolicy — should now succeed (was returning XML parsing error)
|
||||
_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
|
||||
require.NoError(t, err, "DeletePolicy should succeed after detach (was returning XML parsing error)")
|
||||
|
||||
// Step 11: Verify the policy is gone
|
||||
listOut, err = iamClient.ListPolicies(&iam.ListPoliciesInput{})
|
||||
require.NoError(t, err)
|
||||
require.False(t, managedPolicyContains(listOut.Policies, policyName),
|
||||
"Deleted policy should not appear in ListPolicies")
|
||||
|
||||
_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
|
||||
require.Error(t, err, "GetPolicy should fail for deleted policy")
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
}
|
||||
|
||||
// TestS3IAMManagedPolicyErrorCases covers error cases from the user-reported issue:
|
||||
// invalid ARNs, missing policies, and missing users.
|
||||
func TestS3IAMManagedPolicyErrorCases(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
iamClient := newIAMClient(t, cluster.s3Endpoint)
|
||||
|
||||
t.Run("GetPolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
|
||||
_, err := iamClient.GetPolicy(&iam.GetPolicyInput{
|
||||
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("DeletePolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
|
||||
_, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{
|
||||
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("AttachUserPolicy with nonexistent policy returns NoSuchEntity", func(t *testing.T) {
|
||||
userName := uniqueName("err-user")
|
||||
_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("AttachUserPolicy with nonexistent user returns NoSuchEntity", func(t *testing.T) {
|
||||
policyName := uniqueName("err-policy")
|
||||
_, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String("nonexistent-user"),
|
||||
PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("DetachUserPolicy that is not attached returns NoSuchEntity", func(t *testing.T) {
|
||||
userName := uniqueName("detach-user")
|
||||
_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
policyName := uniqueName("detach-policy")
|
||||
_, err = iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("ListAttachedUserPolicies for nonexistent user returns NoSuchEntity", func(t *testing.T) {
|
||||
_, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
|
||||
UserName: aws.String("nonexistent-user"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
}
|
||||
|
||||
func execShell(t *testing.T, weedCmd, master, filer, shellCmd string) string {
|
||||
// weed shell -master=... -filer=...
|
||||
args := []string{"shell", "-master=" + master, "-filer=" + filer}
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
FROM alpine:3.20
|
||||
RUN apk add --no-cache curl && \
|
||||
addgroup -S seaweed && \
|
||||
adduser -S seaweed -G seaweed
|
||||
COPY weed /usr/bin/weed
|
||||
RUN chmod +x /usr/bin/weed && \
|
||||
chown seaweed:seaweed /usr/bin/weed && \
|
||||
mkdir -p /etc/seaweedfs /data/filerldb2 && \
|
||||
chown -R seaweed:seaweed /etc/seaweedfs /data && \
|
||||
chmod 755 /data /etc/seaweedfs /data/filerldb2
|
||||
WORKDIR /data
|
||||
USER seaweed
|
||||
@@ -1,79 +0,0 @@
|
||||
# S3 Proxy Signature Verification Test
|
||||
|
||||
Integration test that verifies S3 signature verification works correctly when
|
||||
SeaweedFS is deployed behind a reverse proxy (nginx).
|
||||
|
||||
## What it tests
|
||||
|
||||
- S3 operations (create bucket, put/get/head/list/delete) through an nginx
|
||||
reverse proxy with `X-Forwarded-Host`, `X-Forwarded-Port`, and
|
||||
`X-Forwarded-Proto` headers
|
||||
- SeaweedFS configured with `-s3.externalUrl=http://localhost:9000` so the
|
||||
signature verification uses the client-facing host instead of the internal
|
||||
backend address
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
AWS CLI (signs with Host: localhost:9000)
|
||||
|
|
||||
v
|
||||
nginx (:9000)
|
||||
| proxy_pass → seaweedfs:8333
|
||||
| Sets: X-Forwarded-Host: localhost
|
||||
| X-Forwarded-Port: 9000
|
||||
| X-Forwarded-Proto: http
|
||||
v
|
||||
SeaweedFS S3 (:8333, -s3.externalUrl=http://localhost:9000)
|
||||
| externalHost = "localhost:9000" (parsed at startup)
|
||||
| extractHostHeader() returns "localhost:9000"
|
||||
| Matches what AWS CLI signed with
|
||||
v
|
||||
Signature verification succeeds
|
||||
```
|
||||
|
||||
**Note:** When `-s3.externalUrl` is configured, direct access to the backend
|
||||
port (8333) will fail signature verification because the client signs with a
|
||||
different Host header than what `externalUrl` specifies. This is expected —
|
||||
all S3 traffic should go through the proxy.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker and Docker Compose
|
||||
- AWS CLI v2 (on host or via Docker, see below)
|
||||
|
||||
## Running
|
||||
|
||||
```bash
|
||||
# Build the weed binary first (from repo root):
|
||||
cd /path/to/seaweedfs
|
||||
go build -o test/s3/proxy_signature/weed ./weed
|
||||
cd test/s3/proxy_signature
|
||||
|
||||
# Start services
|
||||
docker compose up -d --build
|
||||
|
||||
# Option A: Run test with aws CLI installed locally
|
||||
./test.sh
|
||||
|
||||
# Option B: Run test without aws CLI (uses Docker container)
|
||||
docker run --rm --network host --entrypoint "" amazon/aws-cli:latest \
|
||||
bash < test.sh
|
||||
|
||||
# Tear down
|
||||
docker compose down
|
||||
|
||||
# Clean up the weed binary
|
||||
rm -f weed
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
If signature verification fails through the proxy, check:
|
||||
|
||||
1. nginx is setting `X-Forwarded-Host` and `X-Forwarded-Port` correctly
|
||||
2. SeaweedFS is started with `-s3.externalUrl` matching the client endpoint
|
||||
3. The AWS CLI endpoint URL matches the proxy address
|
||||
|
||||
You can also set the `S3_EXTERNAL_URL` environment variable instead of the
|
||||
`-s3.externalUrl` flag.
|
||||
@@ -1,28 +0,0 @@
|
||||
services:
|
||||
seaweedfs:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: >
|
||||
/usr/bin/weed mini
|
||||
-s3.config=/etc/seaweedfs/s3.json
|
||||
-s3.externalUrl=http://localhost:9000
|
||||
-ip=seaweedfs
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json:ro
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://seaweedfs:8333/status"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
|
||||
nginx:
|
||||
image: nginx:alpine
|
||||
ports:
|
||||
- "9000:9000"
|
||||
volumes:
|
||||
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
|
||||
depends_on:
|
||||
seaweedfs:
|
||||
condition: service_healthy
|
||||
@@ -1,23 +0,0 @@
|
||||
server {
|
||||
listen 9000;
|
||||
server_name localhost;
|
||||
|
||||
# Allow large uploads
|
||||
client_max_body_size 64m;
|
||||
|
||||
location / {
|
||||
proxy_pass http://seaweedfs:8333;
|
||||
|
||||
# Standard reverse proxy headers — this is what Kong, Traefik, etc. do
|
||||
proxy_set_header Host $host:$server_port;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Host $host;
|
||||
proxy_set_header X-Forwarded-Port $server_port;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "test_admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "test_access_key",
|
||||
"secretKey": "test_secret_key"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Integration test for S3 signature verification behind a reverse proxy.
|
||||
#
|
||||
# Usage:
|
||||
# # With aws CLI installed locally:
|
||||
# docker compose up -d --build && ./test.sh && docker compose down
|
||||
#
|
||||
# # Without aws CLI (runs test inside a container):
|
||||
# docker compose up -d --build
|
||||
# docker run --rm --network host --entrypoint "" amazon/aws-cli:latest \
|
||||
# bash < test.sh
|
||||
# docker compose down
|
||||
#
|
||||
# This script tests S3 operations through an nginx reverse proxy to verify
|
||||
# that signature verification works correctly when SeaweedFS is configured
|
||||
# with -s3.externalUrl=http://localhost:9000.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
PROXY_ENDPOINT="http://localhost:9000"
|
||||
ACCESS_KEY="test_access_key"
|
||||
SECRET_KEY="test_secret_key"
|
||||
REGION="us-east-1"
|
||||
BUCKET="test-proxy-sig-$$"
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
NC='\033[0m'
|
||||
|
||||
pass() { echo -e "${GREEN}PASS${NC}: $1"; }
|
||||
fail() { echo -e "${RED}FAIL${NC}: $1"; exit 1; }
|
||||
|
||||
# Helper: run aws s3api command against a given endpoint
|
||||
s3() {
|
||||
local endpoint="$1"
|
||||
shift
|
||||
aws s3api \
|
||||
--endpoint-url "$endpoint" \
|
||||
--region "$REGION" \
|
||||
--no-verify-ssl \
|
||||
"$@" 2>&1
|
||||
}
|
||||
|
||||
export AWS_ACCESS_KEY_ID="$ACCESS_KEY"
|
||||
export AWS_SECRET_ACCESS_KEY="$SECRET_KEY"
|
||||
|
||||
echo "=== S3 Proxy Signature Verification Test ==="
|
||||
echo ""
|
||||
echo "Testing S3 access through nginx reverse proxy at $PROXY_ENDPOINT"
|
||||
echo "SeaweedFS configured with -s3.externalUrl=http://localhost:9000"
|
||||
echo "AWS CLI signs requests with Host: localhost:9000"
|
||||
echo ""
|
||||
|
||||
# Wait for proxy to be ready
|
||||
echo "Waiting for nginx proxy to be ready..."
|
||||
for i in $(seq 1 30); do
|
||||
# Use aws CLI for health check if curl is missing
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
http_code=$(curl -s -o /dev/null -w "%{http_code}" "$PROXY_ENDPOINT/" 2>/dev/null || echo "000")
|
||||
case $http_code in
|
||||
200|403|405) break ;;
|
||||
esac
|
||||
else
|
||||
if aws s3api list-buckets --endpoint-url "$PROXY_ENDPOINT" --no-sign-request >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
fi
|
||||
if [ "$i" -eq 30 ]; then
|
||||
fail "Proxy did not become ready in time"
|
||||
fi
|
||||
echo "Waiting for proxy $i/30..."
|
||||
sleep 1
|
||||
done
|
||||
echo "Proxy is ready."
|
||||
echo ""
|
||||
|
||||
# --- Test 1: Bucket operations through proxy ---
|
||||
echo "--- Test 1: Bucket operations through proxy ---"
|
||||
s3 "$PROXY_ENDPOINT" create-bucket --bucket "$BUCKET" > /dev/null \
|
||||
&& pass "create-bucket" \
|
||||
|| fail "create-bucket — signature verification likely failed"
|
||||
|
||||
s3 "$PROXY_ENDPOINT" list-buckets > /dev/null \
|
||||
&& pass "list-buckets" \
|
||||
|| fail "list-buckets"
|
||||
echo ""
|
||||
|
||||
# --- Test 2: Object CRUD through proxy ---
|
||||
echo "--- Test 2: Object CRUD through proxy ---"
|
||||
echo "hello-from-proxy" > /tmp/test-proxy-sig.txt
|
||||
|
||||
s3 "$PROXY_ENDPOINT" put-object --bucket "$BUCKET" --key "test.txt" --body /tmp/test-proxy-sig.txt > /dev/null \
|
||||
&& pass "put-object" \
|
||||
|| fail "put-object"
|
||||
|
||||
s3 "$PROXY_ENDPOINT" head-object --bucket "$BUCKET" --key "test.txt" > /dev/null \
|
||||
&& pass "head-object" \
|
||||
|| fail "head-object"
|
||||
|
||||
s3 "$PROXY_ENDPOINT" list-objects-v2 --bucket "$BUCKET" > /dev/null \
|
||||
&& pass "list-objects-v2" \
|
||||
|| fail "list-objects-v2"
|
||||
|
||||
s3 "$PROXY_ENDPOINT" get-object --bucket "$BUCKET" --key "test.txt" /tmp/test-proxy-sig-get.txt > /dev/null \
|
||||
&& pass "get-object" \
|
||||
|| fail "get-object"
|
||||
|
||||
# Verify content round-trip
|
||||
CONTENT=$(cat /tmp/test-proxy-sig-get.txt)
|
||||
if [ "$CONTENT" = "hello-from-proxy" ]; then
|
||||
pass "content integrity (round-trip)"
|
||||
else
|
||||
fail "content mismatch: got \"$CONTENT\", expected \"hello-from-proxy\""
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# --- Test 3: Delete operations through proxy ---
|
||||
echo "--- Test 3: Delete through proxy ---"
|
||||
s3 "$PROXY_ENDPOINT" delete-object --bucket "$BUCKET" --key "test.txt" > /dev/null \
|
||||
&& pass "delete-object" \
|
||||
|| fail "delete-object"
|
||||
|
||||
s3 "$PROXY_ENDPOINT" delete-bucket --bucket "$BUCKET" > /dev/null \
|
||||
&& pass "delete-bucket" \
|
||||
|| fail "delete-bucket"
|
||||
echo ""
|
||||
|
||||
# Cleanup temp files
|
||||
rm -f /tmp/test-proxy-sig.txt /tmp/test-proxy-sig-get.txt
|
||||
|
||||
echo "=== All tests passed ==="
|
||||
@@ -1,32 +1,15 @@
|
||||
package lakekeeper
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/aws-sdk-go-v2/service/sts"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
type TestEnvironment struct {
|
||||
@@ -48,19 +31,15 @@ type TestEnvironment struct {
|
||||
secretKey string
|
||||
}
|
||||
|
||||
const (
|
||||
lakekeeperRegion = "us-east-1"
|
||||
lakekeeperRoleArn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
|
||||
lakekeeperSessionName = "lakekeeper-session"
|
||||
)
|
||||
|
||||
type lakekeeperSession struct {
|
||||
endpoint string
|
||||
region string
|
||||
creds aws.Credentials
|
||||
}
|
||||
|
||||
func TestLakekeeperIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
if !testutil.HasDocker() {
|
||||
t.Skip("Docker not available, skipping Lakekeeper integration test")
|
||||
}
|
||||
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
@@ -72,18 +51,6 @@ func TestLakekeeperIntegration(t *testing.T) {
|
||||
runLakekeeperRepro(t, env)
|
||||
}
|
||||
|
||||
func TestLakekeeperTableBucketIntegration(t *testing.T) {
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
fmt.Printf(">>> Starting SeaweedFS with Lakekeeper configuration...\n")
|
||||
env.StartSeaweedFS(t)
|
||||
fmt.Printf(">>> SeaweedFS started.\n")
|
||||
|
||||
// Run python script in docker to test STS and S3 Tables operations
|
||||
runLakekeeperTableBucketRepro(t, env)
|
||||
}
|
||||
|
||||
func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
t.Helper()
|
||||
|
||||
@@ -251,439 +218,124 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
|
||||
func runLakekeeperRepro(t *testing.T, env *TestEnvironment) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
scriptContent := fmt.Sprintf(`
|
||||
import boto3
|
||||
import botocore.config
|
||||
import botocore
|
||||
from botocore.exceptions import ClientError
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
|
||||
session, err := newLakekeeperSession(ctx, env)
|
||||
if err != nil {
|
||||
t.Fatalf("AssumeRole failed: %v", err)
|
||||
# Enable botocore debug logging to see signature calculation
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
botocore.session.get_session().set_debug_logger()
|
||||
|
||||
print("Starting Lakekeeper repro test...")
|
||||
|
||||
endpoint_url = "http://host.docker.internal:%d"
|
||||
access_key = "%s"
|
||||
secret_key = "%s"
|
||||
region = "us-east-1"
|
||||
|
||||
print(f"Connecting to {endpoint_url}")
|
||||
|
||||
try:
|
||||
config = botocore.config.Config(
|
||||
retries={'max_attempts': 3}
|
||||
)
|
||||
sts = boto3.client(
|
||||
'sts',
|
||||
endpoint_url=endpoint_url,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=region,
|
||||
config=config
|
||||
)
|
||||
|
||||
role_arn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
|
||||
session_name = "lakekeeper-session"
|
||||
|
||||
print(f"Calling AssumeRole on {role_arn} with POST body...")
|
||||
|
||||
# Standard boto3 call sends parameters in POST body
|
||||
response = sts.assume_role(
|
||||
RoleArn=role_arn,
|
||||
RoleSessionName=session_name
|
||||
)
|
||||
|
||||
creds = response['Credentials']
|
||||
access_key_id = creds['AccessKeyId']
|
||||
secret_access_key = creds['SecretAccessKey']
|
||||
session_token = creds['SessionToken']
|
||||
|
||||
print(f"Success! Got credentials with prefix: {access_key_id[:4]}")
|
||||
|
||||
if not access_key_id.startswith("ASIA"):
|
||||
print(f"FAILED: Expected ASIA prefix, got {access_key_id}")
|
||||
sys.exit(1)
|
||||
|
||||
print("Verifying S3 operations with vended credentials...")
|
||||
s3 = boto3.client(
|
||||
's3',
|
||||
endpoint_url=endpoint_url,
|
||||
aws_access_key_id=access_key_id,
|
||||
aws_secret_access_key=secret_access_key,
|
||||
aws_session_token=session_token,
|
||||
region_name=region,
|
||||
config=config
|
||||
)
|
||||
|
||||
bucket = "lakekeeper-vended-bucket"
|
||||
print(f"Creating bucket {bucket}...")
|
||||
s3.create_bucket(Bucket=bucket)
|
||||
|
||||
print("Listing buckets...")
|
||||
response = s3.list_buckets()
|
||||
buckets = [b['Name'] for b in response['Buckets']]
|
||||
print(f"Found buckets: {buckets}")
|
||||
|
||||
if bucket not in buckets:
|
||||
print(f"FAILED: Bucket {bucket} not found in list")
|
||||
sys.exit(1)
|
||||
|
||||
print("SUCCESS: Lakekeeper flow verified!")
|
||||
sys.exit(0)
|
||||
|
||||
except Exception as e:
|
||||
print(f"FAILED: {e}")
|
||||
# Print more details if it is a ClientError
|
||||
if hasattr(e, 'response'):
|
||||
print(f"Response: {e.response}")
|
||||
sys.exit(1)
|
||||
`, env.s3Port, env.accessKey, env.secretKey)
|
||||
|
||||
scriptPath := filepath.Join(env.dataDir, "lakekeeper_repro.py")
|
||||
if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
|
||||
t.Fatalf("Failed to write python script: %v", err)
|
||||
}
|
||||
|
||||
s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
|
||||
if err != nil {
|
||||
t.Fatalf("Create S3 client failed: %v", err)
|
||||
}
|
||||
containerName := "seaweed-lakekeeper-client-" + fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
|
||||
bucketName := fmt.Sprintf("lakekeeper-vended-bucket-%d", time.Now().UnixNano())
|
||||
if _, err := s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateBucket failed: %v", err)
|
||||
}
|
||||
// Create a context with timeout for the docker run command
|
||||
dockerCtx, dockerCancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer dockerCancel()
|
||||
|
||||
bucketCreated := true
|
||||
defer func() {
|
||||
if !bucketCreated {
|
||||
return
|
||||
}
|
||||
_, _ = s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
}()
|
||||
|
||||
listResp, err := s3Client.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
if err != nil {
|
||||
t.Fatalf("ListBuckets failed: %v", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, bucket := range listResp.Buckets {
|
||||
if aws.ToString(bucket.Name) == bucketName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("Bucket %s not found in list", bucketName)
|
||||
}
|
||||
|
||||
if _, err := s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
}); err != nil {
|
||||
t.Fatalf("DeleteBucket failed: %v", err)
|
||||
}
|
||||
bucketCreated = false
|
||||
}
|
||||
|
||||
func runLakekeeperTableBucketRepro(t *testing.T, env *TestEnvironment) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
session, err := newLakekeeperSession(ctx, env)
|
||||
if err != nil {
|
||||
t.Fatalf("AssumeRole failed: %v", err)
|
||||
}
|
||||
|
||||
client := newS3TablesClient(session.endpoint, session.region, session.creds)
|
||||
bucketName := fmt.Sprintf("lakekeeper-table-bucket-%d", time.Now().UnixNano())
|
||||
bucketARN, err := client.CreateTableBucket(ctx, bucketName)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateTableBucket failed: %v", err)
|
||||
}
|
||||
|
||||
bucketDeleted := false
|
||||
namespaceCreated := false
|
||||
tableCreated := false
|
||||
namespaceName := fmt.Sprintf("lakekeeper_ns_%d", time.Now().UnixNano())
|
||||
tableName := fmt.Sprintf("lakekeeper_table_%d", time.Now().UnixNano())
|
||||
defer func() {
|
||||
if tableCreated {
|
||||
if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
|
||||
t.Logf("Failed to delete table: %v", err)
|
||||
}
|
||||
}
|
||||
if namespaceCreated {
|
||||
if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
|
||||
t.Logf("Failed to delete namespace: %v", err)
|
||||
}
|
||||
}
|
||||
if bucketDeleted {
|
||||
return
|
||||
}
|
||||
if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
|
||||
t.Logf("Failed to delete table bucket: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
buckets, err := client.ListTableBuckets(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTableBuckets failed: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, b := range buckets {
|
||||
if b.Name == bucketName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("Created table bucket %s not found in list", bucketName)
|
||||
}
|
||||
|
||||
if _, err := client.GetTableBucket(ctx, bucketARN); err != nil {
|
||||
t.Fatalf("GetTableBucket failed: %v", err)
|
||||
}
|
||||
|
||||
if err := client.CreateNamespace(ctx, bucketARN, namespaceName); err != nil {
|
||||
t.Fatalf("CreateNamespace failed: %v", err)
|
||||
}
|
||||
namespaceCreated = true
|
||||
|
||||
if err := client.CreateTable(ctx, bucketARN, namespaceName, tableName); err != nil {
|
||||
t.Fatalf("CreateTable failed: %v", err)
|
||||
}
|
||||
tableCreated = true
|
||||
|
||||
s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
|
||||
if err != nil {
|
||||
t.Fatalf("Create S3 client failed: %v", err)
|
||||
}
|
||||
|
||||
objectKey := fmt.Sprintf("%s/%s/data/part-%d.parquet", namespaceName, tableName, time.Now().UnixNano())
|
||||
createResp, err := s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateMultipartUpload failed: %v", err)
|
||||
}
|
||||
uploadID := aws.ToString(createResp.UploadId)
|
||||
multipartCompleted := false
|
||||
defer func() {
|
||||
if uploadID == "" || multipartCompleted {
|
||||
return
|
||||
}
|
||||
_, _ = s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
})
|
||||
}()
|
||||
|
||||
partSize := 5 * 1024 * 1024
|
||||
part1 := bytes.Repeat([]byte("a"), partSize)
|
||||
part2 := bytes.Repeat([]byte("b"), 1024*1024)
|
||||
|
||||
part1Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader(part1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UploadPart 1 failed: %v", err)
|
||||
}
|
||||
|
||||
part2Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
PartNumber: aws.Int32(2),
|
||||
Body: bytes.NewReader(part2),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UploadPart 2 failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
MultipartUpload: &s3types.CompletedMultipartUpload{
|
||||
Parts: []s3types.CompletedPart{
|
||||
{
|
||||
ETag: part1Resp.ETag,
|
||||
PartNumber: aws.Int32(1),
|
||||
},
|
||||
{
|
||||
ETag: part2Resp.ETag,
|
||||
PartNumber: aws.Int32(2),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteMultipartUpload failed: %v", err)
|
||||
}
|
||||
multipartCompleted = true
|
||||
|
||||
headResp, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("HeadObject after multipart upload failed: %v", err)
|
||||
}
|
||||
expectedSize := int64(len(part1) + len(part2))
|
||||
if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
|
||||
t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
|
||||
}
|
||||
|
||||
if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
|
||||
t.Fatalf("DeleteTable failed: %v", err)
|
||||
}
|
||||
tableCreated = false
|
||||
|
||||
if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
|
||||
t.Fatalf("DeleteNamespace failed: %v", err)
|
||||
}
|
||||
namespaceCreated = false
|
||||
|
||||
if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
|
||||
t.Fatalf("DeleteTableBucket failed: %v", err)
|
||||
}
|
||||
bucketDeleted = true
|
||||
|
||||
if _, err := client.GetTableBucket(ctx, bucketARN); err == nil {
|
||||
t.Fatalf("expected GetTableBucket to fail after deletion")
|
||||
}
|
||||
}
|
||||
|
||||
type s3TablesClient struct {
|
||||
endpoint string
|
||||
region string
|
||||
creds aws.Credentials
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func newS3TablesClient(endpoint, region string, creds aws.Credentials) *s3TablesClient {
|
||||
return &s3TablesClient{
|
||||
endpoint: endpoint,
|
||||
region: region,
|
||||
creds: creds,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) CreateTableBucket(ctx context.Context, name string) (string, error) {
|
||||
req := &s3tables.CreateTableBucketRequest{Name: name}
|
||||
var resp s3tables.CreateTableBucketResponse
|
||||
if err := c.doRequest(ctx, "CreateTableBucket", http.MethodPut, "/buckets", req, &resp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resp.ARN, nil
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) GetTableBucket(ctx context.Context, arn string) (*s3tables.GetTableBucketResponse, error) {
|
||||
path := "/buckets/" + url.PathEscape(arn)
|
||||
var resp s3tables.GetTableBucketResponse
|
||||
if err := c.doRequest(ctx, "GetTableBucket", http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) ListTableBuckets(ctx context.Context) ([]s3tables.TableBucketSummary, error) {
|
||||
var resp s3tables.ListTableBucketsResponse
|
||||
if err := c.doRequest(ctx, "ListTableBuckets", http.MethodGet, "/buckets", nil, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resp.TableBuckets, nil
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) DeleteTableBucket(ctx context.Context, arn string) error {
|
||||
path := "/buckets/" + url.PathEscape(arn)
|
||||
return c.doRequest(ctx, "DeleteTableBucket", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) CreateNamespace(ctx context.Context, bucketARN, namespace string) error {
|
||||
req := &s3tables.CreateNamespaceRequest{
|
||||
Namespace: []string{namespace},
|
||||
}
|
||||
path := "/namespaces/" + url.PathEscape(bucketARN)
|
||||
return c.doRequest(ctx, "CreateNamespace", http.MethodPut, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) DeleteNamespace(ctx context.Context, bucketARN, namespace string) error {
|
||||
path := "/namespaces/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
|
||||
return c.doRequest(ctx, "DeleteNamespace", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) CreateTable(ctx context.Context, bucketARN, namespace, name string) error {
|
||||
req := &s3tables.CreateTableRequest{
|
||||
Name: name,
|
||||
Format: "ICEBERG",
|
||||
}
|
||||
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
|
||||
return c.doRequest(ctx, "CreateTable", http.MethodPut, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) DeleteTable(ctx context.Context, bucketARN, namespace, name string) error {
|
||||
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace) + "/" + url.PathEscape(name)
|
||||
return c.doRequest(ctx, "DeleteTable", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) doRequest(ctx context.Context, operation, method, path string, body interface{}, out interface{}) error {
|
||||
var bodyBytes []byte
|
||||
if body != nil {
|
||||
encoded, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: marshal request: %w", operation, err)
|
||||
}
|
||||
bodyBytes = encoded
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.endpoint+path, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: create request: %w", operation, err)
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
|
||||
}
|
||||
req.Host = req.URL.Host
|
||||
req.Header.Set("Host", req.URL.Host)
|
||||
|
||||
payloadHash := sha256.Sum256(bodyBytes)
|
||||
if err := v4.NewSigner().SignHTTP(ctx, c.creds, req, hex.EncodeToString(payloadHash[:]), "s3tables", c.region, time.Now()); err != nil {
|
||||
return fmt.Errorf("%s: sign request: %w", operation, err)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: request failed: %w", operation, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("%s failed with status %d and could not read response body: %v", operation, resp.StatusCode, readErr)
|
||||
}
|
||||
var errResp s3tables.S3TablesError
|
||||
if jsonErr := json.Unmarshal(bodyBytes, &errResp); jsonErr == nil && (errResp.Type != "" || errResp.Message != "") {
|
||||
return fmt.Errorf("%s failed: %s - %s", operation, errResp.Type, errResp.Message)
|
||||
}
|
||||
return fmt.Errorf("%s failed with status %d: %s", operation, resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
|
||||
}
|
||||
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
|
||||
return fmt.Errorf("%s: decode response: %w", operation, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func assumeRole(ctx context.Context, endpoint, region, accessKey, secretKey, roleArn, sessionName string) (aws.Credentials, error) {
|
||||
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
if service == sts.ServiceID {
|
||||
return aws.Endpoint{
|
||||
URL: endpoint,
|
||||
SigningRegion: region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
|
||||
})
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
|
||||
config.WithEndpointResolverWithOptions(resolver),
|
||||
cmd := exec.CommandContext(dockerCtx, "docker", "run", "--rm",
|
||||
"--name", containerName,
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-v", fmt.Sprintf("%s:/work", env.dataDir),
|
||||
"python:3",
|
||||
"/bin/bash", "-c", "pip install boto3 && python /work/lakekeeper_repro.py",
|
||||
)
|
||||
if err != nil {
|
||||
return aws.Credentials{}, err
|
||||
}
|
||||
|
||||
client := sts.NewFromConfig(cfg)
|
||||
resp, err := client.AssumeRole(ctx, &sts.AssumeRoleInput{
|
||||
RoleArn: aws.String(roleArn),
|
||||
RoleSessionName: aws.String(sessionName),
|
||||
})
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return aws.Credentials{}, err
|
||||
}
|
||||
if resp.Credentials == nil {
|
||||
return aws.Credentials{}, fmt.Errorf("missing credentials in AssumeRole response")
|
||||
}
|
||||
return aws.Credentials{
|
||||
AccessKeyID: aws.ToString(resp.Credentials.AccessKeyId),
|
||||
SecretAccessKey: aws.ToString(resp.Credentials.SecretAccessKey),
|
||||
SessionToken: aws.ToString(resp.Credentials.SessionToken),
|
||||
Source: "lakekeeper-sts",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials) (*s3.Client, error) {
|
||||
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
if service == s3.ServiceID {
|
||||
return aws.Endpoint{
|
||||
URL: endpoint,
|
||||
SigningRegion: region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
if dockerCtx.Err() == context.DeadlineExceeded {
|
||||
t.Fatalf("Lakekeeper repro client timed out after 5 minutes\nOutput:\n%s", string(output))
|
||||
}
|
||||
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
|
||||
})
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
|
||||
config.WithEndpointResolverWithOptions(resolver),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
t.Fatalf("Lakekeeper repro client failed: %v\nOutput:\n%s", err, string(output))
|
||||
}
|
||||
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.UsePathStyle = true
|
||||
}), nil
|
||||
}
|
||||
|
||||
func newLakekeeperSession(ctx context.Context, env *TestEnvironment) (lakekeeperSession, error) {
|
||||
endpoint := fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
|
||||
creds, err := assumeRole(ctx, endpoint, lakekeeperRegion, env.accessKey, env.secretKey, lakekeeperRoleArn, lakekeeperSessionName)
|
||||
if err != nil {
|
||||
return lakekeeperSession{}, err
|
||||
}
|
||||
return lakekeeperSession{
|
||||
endpoint: endpoint,
|
||||
region: lakekeeperRegion,
|
||||
creds: creds,
|
||||
}, nil
|
||||
t.Logf("Lakekeeper repro client output:\n%s", string(output))
|
||||
}
|
||||
|
||||
@@ -1,377 +0,0 @@
|
||||
package polaris
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
)
|
||||
|
||||
const (
|
||||
polarisImage = "apache/polaris:latest"
|
||||
polarisRealm = "POLARIS"
|
||||
polarisRootClientID = "root"
|
||||
polarisRootClientSecret = "s3cr3t"
|
||||
polarisRegion = "us-east-1"
|
||||
polarisRoleArn = "arn:aws:iam::000000000000:role/PolarisVendedRole"
|
||||
polarisSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test signing key
|
||||
)
|
||||
|
||||
type TestEnvironment struct {
|
||||
seaweedDir string
|
||||
weedBinary string
|
||||
dataDir string
|
||||
bindIP string
|
||||
s3Port int
|
||||
s3GrpcPort int
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
filerPort int
|
||||
filerGrpcPort int
|
||||
volumePort int
|
||||
volumeGrpcPort int
|
||||
polarisPort int
|
||||
polarisAdminPort int
|
||||
weedProcess *exec.Cmd
|
||||
weedCancel context.CancelFunc
|
||||
polarisContainer string
|
||||
accessKey string
|
||||
secretKey string
|
||||
}
|
||||
|
||||
func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
t.Helper()
|
||||
|
||||
if !testutil.HasDocker() {
|
||||
t.Skip("Docker is required for Polaris integration tests")
|
||||
}
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get working directory: %v", err)
|
||||
}
|
||||
|
||||
seaweedDir := wd
|
||||
for i := 0; i < 6; i++ {
|
||||
if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
|
||||
break
|
||||
}
|
||||
seaweedDir = filepath.Dir(seaweedDir)
|
||||
}
|
||||
|
||||
weedBinary := filepath.Join(seaweedDir, "weed", "weed")
|
||||
if _, err := os.Stat(weedBinary); err != nil {
|
||||
weedBinary = "weed"
|
||||
if _, err := exec.LookPath(weedBinary); err != nil {
|
||||
t.Skip("weed binary not found, skipping integration test")
|
||||
}
|
||||
}
|
||||
|
||||
dataDir, err := os.MkdirTemp("", "seaweed-polaris-test-*")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
|
||||
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
|
||||
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
|
||||
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
|
||||
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
|
||||
polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
|
||||
|
||||
return &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
bindIP: bindIP,
|
||||
s3Port: s3Port,
|
||||
s3GrpcPort: s3GrpcPort,
|
||||
masterPort: masterPort,
|
||||
masterGrpcPort: masterGrpcPort,
|
||||
filerPort: filerPort,
|
||||
filerGrpcPort: filerGrpcPort,
|
||||
volumePort: volumePort,
|
||||
volumeGrpcPort: volumeGrpcPort,
|
||||
polarisPort: polarisPort,
|
||||
polarisAdminPort: polarisAdminPort,
|
||||
accessKey: "admin",
|
||||
secretKey: "admin",
|
||||
}
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
iamConfigPath := filepath.Join(env.dataDir, "iam.json")
|
||||
iamConfig := fmt.Sprintf(`{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "%s",
|
||||
"secretKey": "%s"
|
||||
}
|
||||
],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
|
||||
}
|
||||
],
|
||||
"sts": {
|
||||
"tokenDuration": "12h",
|
||||
"maxSessionLength": "24h",
|
||||
"issuer": "seaweedfs-sts",
|
||||
"signingKey": "%s"
|
||||
},
|
||||
"roles": [
|
||||
{
|
||||
"roleName": "PolarisVendedRole",
|
||||
"roleArn": "%s",
|
||||
"trustPolicy": {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "sts:AssumeRole"
|
||||
}
|
||||
]
|
||||
},
|
||||
"attachedPolicies": ["FullAccess"]
|
||||
}
|
||||
],
|
||||
"policies": [
|
||||
{
|
||||
"name": "FullAccess",
|
||||
"document": {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": "*",
|
||||
"Resource": "*"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}`, env.accessKey, env.secretKey, polarisSigningKey, polarisRoleArn)
|
||||
|
||||
if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
|
||||
t.Fatalf("Failed to create IAM config: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
env.weedCancel = cancel
|
||||
|
||||
cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini",
|
||||
"-master.port", fmt.Sprintf("%d", env.masterPort),
|
||||
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
|
||||
"-volume.port", fmt.Sprintf("%d", env.volumePort),
|
||||
"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
|
||||
"-filer.port", fmt.Sprintf("%d", env.filerPort),
|
||||
"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
|
||||
"-s3.port", fmt.Sprintf("%d", env.s3Port),
|
||||
"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
|
||||
"-s3.config", iamConfigPath,
|
||||
"-s3.iam.config", iamConfigPath,
|
||||
"-s3.iam.readOnly=false",
|
||||
"-ip", env.bindIP,
|
||||
"-ip.bind", "0.0.0.0",
|
||||
"-dir", env.dataDir,
|
||||
)
|
||||
cmd.Dir = env.dataDir
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("Failed to start SeaweedFS: %v", err)
|
||||
}
|
||||
env.weedProcess = cmd
|
||||
|
||||
if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
|
||||
t.Fatalf("S3 API failed to become ready")
|
||||
}
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) StartPolaris(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
containerName := fmt.Sprintf("seaweed-polaris-%d", time.Now().UnixNano())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, "docker", "run", "-d", "--rm",
|
||||
"--name", containerName,
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-p", fmt.Sprintf("%d:8181", env.polarisPort),
|
||||
"-p", fmt.Sprintf("%d:8182", env.polarisAdminPort),
|
||||
"-e", fmt.Sprintf("AWS_REGION=%s", polarisRegion),
|
||||
"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", env.accessKey),
|
||||
"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", env.secretKey),
|
||||
"-e", fmt.Sprintf("POLARIS_BOOTSTRAP_CREDENTIALS=%s,%s,%s", polarisRealm, polarisRootClientID, polarisRootClientSecret),
|
||||
"-e", fmt.Sprintf("polaris.realm-context.realms=%s", polarisRealm),
|
||||
"-e", "quarkus.otel.sdk.disabled=true",
|
||||
polarisImage,
|
||||
)
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
t.Fatalf("Timed out waiting for Polaris container: %v\nOutput:\n%s", ctx.Err(), string(output))
|
||||
}
|
||||
t.Fatalf("Failed to start Polaris: %v\nOutput:\n%s", err, string(output))
|
||||
}
|
||||
env.polarisContainer = containerName
|
||||
|
||||
if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/q/health", env.polarisAdminPort), 60*time.Second) {
|
||||
logCtx, logCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer logCancel()
|
||||
logs, _ := exec.CommandContext(logCtx, "docker", "logs", env.polarisContainer).CombinedOutput()
|
||||
t.Fatalf("Polaris failed to become ready\nLogs:\n%s", string(logs))
|
||||
}
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) Cleanup(t *testing.T) {
|
||||
t.Helper()
|
||||
if env.weedCancel != nil {
|
||||
env.weedCancel()
|
||||
}
|
||||
if env.weedProcess != nil {
|
||||
time.Sleep(1 * time.Second)
|
||||
_ = env.weedProcess.Wait()
|
||||
}
|
||||
if env.polarisContainer != "" {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
_ = exec.CommandContext(ctx, "docker", "rm", "-f", env.polarisContainer).Run()
|
||||
}
|
||||
if env.dataDir != "" {
|
||||
_ = os.RemoveAll(env.dataDir)
|
||||
}
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) polarisEndpoint() string {
|
||||
return fmt.Sprintf("http://127.0.0.1:%d", env.polarisPort)
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) s3Endpoint() string {
|
||||
return fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) s3InternalEndpoint() string {
|
||||
return fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
|
||||
}
|
||||
|
||||
type polarisHTTPClient struct {
|
||||
baseURL string
|
||||
realm string
|
||||
token string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func newPolarisHTTPClient(baseURL, realm, token string) *polarisHTTPClient {
|
||||
return &polarisHTTPClient{
|
||||
baseURL: baseURL,
|
||||
realm: realm,
|
||||
token: token,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *polarisHTTPClient) doJSON(ctx context.Context, method, path string, body interface{}, out interface{}) error {
|
||||
return c.doJSONWithHeaders(ctx, method, path, body, out, nil)
|
||||
}
|
||||
|
||||
func (c *polarisHTTPClient) doJSONWithHeaders(ctx context.Context, method, path string, body interface{}, out interface{}, headers map[string]string) error {
|
||||
var reader io.Reader
|
||||
if body != nil {
|
||||
encoded, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode request body: %w", err)
|
||||
}
|
||||
reader = bytes.NewReader(encoded)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if c.realm != "" {
|
||||
req.Header.Set("Polaris-Realm", c.realm)
|
||||
}
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
for key, value := range headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
bodyBytes, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("request failed with status %d and could not read response body: %w", resp.StatusCode, readErr)
|
||||
}
|
||||
return fmt.Errorf("request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials, pathStyle bool) (*s3.Client, error) {
|
||||
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
if service == s3.ServiceID {
|
||||
return aws.Endpoint{
|
||||
URL: endpoint,
|
||||
SigningRegion: region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
|
||||
})
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
|
||||
config.WithEndpointResolverWithOptions(resolver),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.UsePathStyle = pathStyle
|
||||
}), nil
|
||||
}
|
||||
@@ -1,764 +0,0 @@
|
||||
package polaris
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
type polarisSession struct {
|
||||
catalogName string
|
||||
bucketName string
|
||||
token string
|
||||
baseLocation string
|
||||
}
|
||||
|
||||
type polarisTableSetup struct {
|
||||
namespace string
|
||||
table string
|
||||
dataKeyPrefix string
|
||||
s3Client *s3.Client
|
||||
}
|
||||
|
||||
type polarisCatalogClient struct {
|
||||
http *polarisHTTPClient
|
||||
catalog string
|
||||
}
|
||||
|
||||
type polarisCredentials struct {
|
||||
ClientID string `json:"clientId"`
|
||||
ClientSecret string `json:"clientSecret"`
|
||||
}
|
||||
|
||||
type createPrincipalResponse struct {
|
||||
Credentials polarisCredentials `json:"credentials"`
|
||||
}
|
||||
|
||||
type createCatalogRequest struct {
|
||||
Catalog polarisCatalog `json:"catalog"`
|
||||
}
|
||||
|
||||
type polarisCatalog struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
ReadOnly bool `json:"readOnly"`
|
||||
Properties map[string]string `json:"properties"`
|
||||
StorageConfigInfo polarisStorageConfig `json:"storageConfigInfo"`
|
||||
}
|
||||
|
||||
type polarisStorageConfig struct {
|
||||
StorageType string `json:"storageType"`
|
||||
AllowedLocations []string `json:"allowedLocations"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
EndpointInternal string `json:"endpointInternal"`
|
||||
StsEndpoint string `json:"stsEndpoint"`
|
||||
PathStyleAccess bool `json:"pathStyleAccess"`
|
||||
RoleArn string `json:"roleArn"`
|
||||
Region string `json:"region"`
|
||||
}
|
||||
|
||||
type createNamespaceRequest struct {
|
||||
Namespace []string `json:"namespace"`
|
||||
}
|
||||
|
||||
type createTableRequest struct {
|
||||
Name string `json:"name"`
|
||||
Location string `json:"location"`
|
||||
Schema icebergSchema `json:"schema"`
|
||||
PartitionSpec icebergPartition `json:"partition-spec"`
|
||||
SortOrder icebergSortOrder `json:"sort-order"`
|
||||
Properties map[string]string `json:"properties"`
|
||||
}
|
||||
|
||||
type icebergSchema struct {
|
||||
Type string `json:"type"`
|
||||
SchemaID int `json:"schema-id"`
|
||||
Fields []icebergSchemaField `json:"fields"`
|
||||
}
|
||||
|
||||
type icebergSchemaField struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Required bool `json:"required"`
|
||||
}
|
||||
|
||||
type icebergPartition struct {
|
||||
SpecID int `json:"spec-id"`
|
||||
Fields []icebergPartitionField `json:"fields"`
|
||||
}
|
||||
|
||||
type icebergPartitionField struct {
|
||||
SourceID int `json:"source-id"`
|
||||
FieldID int `json:"field-id"`
|
||||
Name string `json:"name"`
|
||||
Transform string `json:"transform"`
|
||||
}
|
||||
|
||||
type icebergSortOrder struct {
|
||||
OrderID int `json:"order-id"`
|
||||
Fields []icebergSortField `json:"fields"`
|
||||
}
|
||||
|
||||
type icebergSortField struct {
|
||||
SourceID int `json:"source-id"`
|
||||
Direction string `json:"direction"`
|
||||
NullOrder string `json:"null-order"`
|
||||
}
|
||||
|
||||
type loadTableResponse struct {
|
||||
Config map[string]string `json:"config"`
|
||||
StorageCredentials []storageCredential `json:"storage-credentials"`
|
||||
}
|
||||
|
||||
type loadCredentialsResponse struct {
|
||||
StorageCredentials []storageCredential `json:"storage-credentials"`
|
||||
}
|
||||
|
||||
type storageCredential struct {
|
||||
Prefix string `json:"prefix"`
|
||||
Config map[string]string `json:"config"`
|
||||
}
|
||||
|
||||
func bootstrapPolarisTest(t *testing.T, env *TestEnvironment) (context.Context, context.CancelFunc, polarisSession, *polarisTableSetup, func()) {
|
||||
t.Helper()
|
||||
|
||||
t.Logf(">>> Starting SeaweedFS with Polaris configuration...")
|
||||
env.StartSeaweedFS(t)
|
||||
t.Logf(">>> SeaweedFS started.")
|
||||
|
||||
t.Logf(">>> Starting Polaris...")
|
||||
env.StartPolaris(t)
|
||||
t.Logf(">>> Polaris started.")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
session := newPolarisSession(t, ctx, env)
|
||||
setup, cleanup := setupPolarisTable(t, ctx, env, session)
|
||||
|
||||
return ctx, cancel, session, setup, cleanup
|
||||
}
|
||||
|
||||
func TestPolarisIntegration(t *testing.T) {
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
|
||||
defer cancel()
|
||||
defer cleanup()
|
||||
|
||||
objectKey := fmt.Sprintf("%s/hello-%d.txt", setup.dataKeyPrefix, time.Now().UnixNano())
|
||||
payload := []byte("polaris")
|
||||
|
||||
if _, err := setup.s3Client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(payload),
|
||||
}); err != nil {
|
||||
t.Fatalf("PutObject failed: %v", err)
|
||||
}
|
||||
|
||||
listObjects, err := setup.s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ListObjectsV2 failed: %v", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, obj := range listObjects.Contents {
|
||||
if aws.ToString(obj.Key) == objectKey {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("Object %s not found in list", objectKey)
|
||||
}
|
||||
|
||||
getResp, err := setup.s3Client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("GetObject failed: %v", err)
|
||||
}
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
_ = getResp.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("Read object body failed: %v", err)
|
||||
}
|
||||
if !bytes.Equal(body, payload) {
|
||||
t.Fatalf("Unexpected object payload: got %q want %q", string(body), string(payload))
|
||||
}
|
||||
|
||||
if _, err := setup.s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
}); err != nil {
|
||||
t.Fatalf("DeleteObject failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolarisTableIntegration(t *testing.T) {
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
|
||||
defer cancel()
|
||||
defer cleanup()
|
||||
|
||||
objectKey := fmt.Sprintf("%s/part-%d.parquet", setup.dataKeyPrefix, time.Now().UnixNano())
|
||||
createResp, err := setup.s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateMultipartUpload failed: %v", err)
|
||||
}
|
||||
|
||||
uploadID := aws.ToString(createResp.UploadId)
|
||||
multipartCompleted := false
|
||||
defer func() {
|
||||
if uploadID == "" || multipartCompleted {
|
||||
return
|
||||
}
|
||||
_, _ = setup.s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
})
|
||||
}()
|
||||
|
||||
partSize := 5 * 1024 * 1024
|
||||
part1 := bytes.Repeat([]byte("a"), partSize)
|
||||
part2 := bytes.Repeat([]byte("b"), 1024*1024)
|
||||
|
||||
part1Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader(part1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UploadPart 1 failed: %v", err)
|
||||
}
|
||||
|
||||
part2Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
PartNumber: aws.Int32(2),
|
||||
Body: bytes.NewReader(part2),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UploadPart 2 failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = setup.s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
MultipartUpload: &s3types.CompletedMultipartUpload{
|
||||
Parts: []s3types.CompletedPart{
|
||||
{
|
||||
ETag: part1Resp.ETag,
|
||||
PartNumber: aws.Int32(1),
|
||||
},
|
||||
{
|
||||
ETag: part2Resp.ETag,
|
||||
PartNumber: aws.Int32(2),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteMultipartUpload failed: %v", err)
|
||||
}
|
||||
multipartCompleted = true
|
||||
|
||||
headResp, err := setup.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("HeadObject after multipart upload failed: %v", err)
|
||||
}
|
||||
expectedSize := int64(len(part1) + len(part2))
|
||||
if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
|
||||
t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
|
||||
}
|
||||
}
|
||||
|
||||
func newPolarisSession(t *testing.T, ctx context.Context, env *TestEnvironment) polarisSession {
|
||||
t.Helper()
|
||||
|
||||
adminCreds := aws.Credentials{
|
||||
AccessKeyID: env.accessKey,
|
||||
SecretAccessKey: env.secretKey,
|
||||
Source: "polaris-admin",
|
||||
}
|
||||
adminS3, err := newS3Client(ctx, env.s3Endpoint(), polarisRegion, adminCreds, true)
|
||||
if err != nil {
|
||||
t.Fatalf("Create admin S3 client failed: %v", err)
|
||||
}
|
||||
|
||||
bucketName := fmt.Sprintf("polaris-bucket-%d", time.Now().UnixNano())
|
||||
if _, err := adminS3.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateBucket failed: %v", err)
|
||||
}
|
||||
policyDoc := map[string]interface{}{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": []map[string]interface{}{
|
||||
{
|
||||
"Sid": "AllowPolarisVendedAccess",
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:*",
|
||||
"Resource": []string{
|
||||
fmt.Sprintf("arn:aws:s3:::%s", bucketName),
|
||||
fmt.Sprintf("arn:aws:s3:::%s/polaris/*", bucketName),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
policyBytes, err := json.Marshal(policyDoc)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal bucket policy: %v", err)
|
||||
}
|
||||
if _, err := adminS3.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Policy: aws.String(string(policyBytes)),
|
||||
}); err != nil {
|
||||
t.Fatalf("PutBucketPolicy failed: %v", err)
|
||||
}
|
||||
|
||||
rootToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), polarisRootClientID, polarisRootClientSecret)
|
||||
if err != nil {
|
||||
t.Fatalf("Polaris root token request failed: %v", err)
|
||||
}
|
||||
|
||||
managementClient := newPolarisHTTPClient(env.polarisEndpoint(), polarisRealm, rootToken)
|
||||
catalogName := fmt.Sprintf("polaris_catalog_%d", time.Now().UnixNano())
|
||||
baseLocation := fmt.Sprintf("s3://%s/polaris", bucketName)
|
||||
|
||||
catalogRequest := createCatalogRequest{
|
||||
Catalog: polarisCatalog{
|
||||
Name: catalogName,
|
||||
Type: "INTERNAL",
|
||||
ReadOnly: false,
|
||||
Properties: map[string]string{
|
||||
"default-base-location": baseLocation,
|
||||
},
|
||||
StorageConfigInfo: polarisStorageConfig{
|
||||
StorageType: "S3",
|
||||
AllowedLocations: []string{baseLocation},
|
||||
Endpoint: env.s3Endpoint(),
|
||||
EndpointInternal: env.s3InternalEndpoint(),
|
||||
StsEndpoint: env.s3InternalEndpoint(),
|
||||
PathStyleAccess: true,
|
||||
RoleArn: polarisRoleArn,
|
||||
Region: polarisRegion,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/catalogs", catalogRequest, nil); err != nil {
|
||||
t.Fatalf("Create catalog failed: %v", err)
|
||||
}
|
||||
|
||||
principalName := fmt.Sprintf("polaris_user_%d", time.Now().UnixNano())
|
||||
principalRoleName := fmt.Sprintf("polaris_principal_role_%d", time.Now().UnixNano())
|
||||
catalogRoleName := fmt.Sprintf("polaris_catalog_role_%d", time.Now().UnixNano())
|
||||
|
||||
var principalResp createPrincipalResponse
|
||||
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principals", map[string]interface{}{
|
||||
"principal": map[string]interface{}{
|
||||
"name": principalName,
|
||||
"properties": map[string]string{},
|
||||
},
|
||||
}, &principalResp); err != nil {
|
||||
t.Fatalf("Create principal failed: %v", err)
|
||||
}
|
||||
if principalResp.Credentials.ClientID == "" || principalResp.Credentials.ClientSecret == "" {
|
||||
t.Fatalf("Missing principal credentials in response")
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principal-roles", map[string]interface{}{
|
||||
"principalRole": map[string]interface{}{
|
||||
"name": principalRoleName,
|
||||
"properties": map[string]string{},
|
||||
},
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("Create principal role failed: %v", err)
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPost, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles", url.PathEscape(catalogName)), map[string]interface{}{
|
||||
"catalogRole": map[string]interface{}{
|
||||
"name": catalogRoleName,
|
||||
"properties": map[string]string{},
|
||||
},
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("Create catalog role failed: %v", err)
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principals/%s/principal-roles", url.PathEscape(principalName)), map[string]interface{}{
|
||||
"principalRole": map[string]interface{}{
|
||||
"name": principalRoleName,
|
||||
},
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("Assign principal role failed: %v", err)
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principal-roles/%s/catalog-roles/%s", url.PathEscape(principalRoleName), url.PathEscape(catalogName)), map[string]interface{}{
|
||||
"catalogRole": map[string]interface{}{
|
||||
"name": catalogRoleName,
|
||||
},
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("Assign catalog role failed: %v", err)
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles/%s/grants", url.PathEscape(catalogName), url.PathEscape(catalogRoleName)), map[string]interface{}{
|
||||
"type": "catalog",
|
||||
"privilege": "CATALOG_MANAGE_CONTENT",
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("Grant catalog privilege failed: %v", err)
|
||||
}
|
||||
|
||||
userToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), principalResp.Credentials.ClientID, principalResp.Credentials.ClientSecret)
|
||||
if err != nil {
|
||||
t.Fatalf("Polaris user token request failed: %v", err)
|
||||
}
|
||||
|
||||
return polarisSession{
|
||||
catalogName: catalogName,
|
||||
bucketName: bucketName,
|
||||
token: userToken,
|
||||
baseLocation: baseLocation,
|
||||
}
|
||||
}
|
||||
|
||||
func setupPolarisTable(t *testing.T, ctx context.Context, env *TestEnvironment, session polarisSession) (*polarisTableSetup, func()) {
|
||||
t.Helper()
|
||||
|
||||
catalogClient := newPolarisCatalogClient(env.polarisEndpoint(), polarisRealm, session.token, session.catalogName)
|
||||
namespace := fmt.Sprintf("polaris_ns_%d", time.Now().UnixNano())
|
||||
table := fmt.Sprintf("polaris_table_%d", time.Now().UnixNano())
|
||||
|
||||
if err := catalogClient.CreateNamespace(ctx, namespace); err != nil {
|
||||
t.Fatalf("CreateNamespace failed: %v", err)
|
||||
}
|
||||
|
||||
location := fmt.Sprintf("%s/%s/%s", session.baseLocation, namespace, table)
|
||||
if err := catalogClient.CreateTable(ctx, namespace, table, location); err != nil {
|
||||
t.Fatalf("CreateTable failed: %v", err)
|
||||
}
|
||||
|
||||
loadResp, err := catalogClient.LoadTable(ctx, namespace, table)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadTable failed: %v", err)
|
||||
}
|
||||
|
||||
credsResp, err := catalogClient.LoadCredentials(ctx, namespace, table)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadCredentials failed: %v", err)
|
||||
}
|
||||
if len(credsResp.StorageCredentials) == 0 {
|
||||
t.Fatalf("LoadCredentials returned no storage credentials")
|
||||
}
|
||||
|
||||
credentialSource := &loadTableResponse{
|
||||
Config: loadResp.Config,
|
||||
StorageCredentials: credsResp.StorageCredentials,
|
||||
}
|
||||
|
||||
dataPrefix := location + "/data"
|
||||
creds, endpoint, region, pathStyle, err := extractS3Credentials(credentialSource, dataPrefix, env.s3Endpoint(), polarisRegion)
|
||||
if err != nil {
|
||||
t.Fatalf("Extract vended credentials failed: %v", err)
|
||||
}
|
||||
dataKeyPrefix, err := s3URIToKeyPrefix(dataPrefix, session.bucketName)
|
||||
if err != nil {
|
||||
t.Fatalf("Invalid data prefix %s: %v", dataPrefix, err)
|
||||
}
|
||||
|
||||
s3Client, err := newS3Client(ctx, endpoint, region, creds, pathStyle)
|
||||
if err != nil {
|
||||
t.Fatalf("Create vended S3 client failed: %v", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cleanupCancel()
|
||||
if err := catalogClient.DeleteTable(cleanupCtx, namespace, table); err != nil {
|
||||
t.Logf("DeleteTable failed: %v", err)
|
||||
}
|
||||
if err := catalogClient.DeleteNamespace(cleanupCtx, namespace); err != nil {
|
||||
t.Logf("DeleteNamespace failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &polarisTableSetup{
|
||||
namespace: namespace,
|
||||
table: table,
|
||||
dataKeyPrefix: dataKeyPrefix,
|
||||
s3Client: s3Client,
|
||||
}, cleanup
|
||||
}
|
||||
|
||||
func fetchPolarisToken(ctx context.Context, baseURL, clientID, clientSecret string) (string, error) {
|
||||
form := url.Values{}
|
||||
form.Set("grant_type", "client_credentials")
|
||||
form.Set("client_id", clientID)
|
||||
form.Set("client_secret", clientSecret)
|
||||
form.Set("scope", "PRINCIPAL_ROLE:ALL")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/api/catalog/v1/oauth/tokens", strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create token request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("token request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
return "", fmt.Errorf("token request failed with status %d and reading body: %w", resp.StatusCode, readErr)
|
||||
}
|
||||
return "", fmt.Errorf("token request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
|
||||
var tokenResp struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
|
||||
return "", fmt.Errorf("decode token response: %w", err)
|
||||
}
|
||||
if tokenResp.AccessToken == "" {
|
||||
return "", fmt.Errorf("missing access token in response")
|
||||
}
|
||||
|
||||
return tokenResp.AccessToken, nil
|
||||
}
|
||||
|
||||
func newPolarisCatalogClient(baseURL, realm, token, catalog string) *polarisCatalogClient {
|
||||
return &polarisCatalogClient{
|
||||
http: newPolarisHTTPClient(baseURL, realm, token),
|
||||
catalog: catalog,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) CreateNamespace(ctx context.Context, namespace string) error {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces", url.PathEscape(c.catalog))
|
||||
req := createNamespaceRequest{Namespace: []string{namespace}}
|
||||
return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) DeleteNamespace(ctx context.Context, namespace string) error {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s", url.PathEscape(c.catalog), url.PathEscape(namespace))
|
||||
return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) CreateTable(ctx context.Context, namespace, table, location string) error {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables", url.PathEscape(c.catalog), url.PathEscape(namespace))
|
||||
|
||||
req := createTableRequest{
|
||||
Name: table,
|
||||
Location: location,
|
||||
Schema: icebergSchema{
|
||||
Type: "struct",
|
||||
SchemaID: 0,
|
||||
Fields: []icebergSchemaField{
|
||||
{
|
||||
ID: 1,
|
||||
Name: "id",
|
||||
Type: "long",
|
||||
Required: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
PartitionSpec: icebergPartition{
|
||||
SpecID: 0,
|
||||
Fields: []icebergPartitionField{},
|
||||
},
|
||||
SortOrder: icebergSortOrder{
|
||||
OrderID: 0,
|
||||
Fields: []icebergSortField{},
|
||||
},
|
||||
Properties: map[string]string{
|
||||
"format-version": "2",
|
||||
},
|
||||
}
|
||||
|
||||
return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) DeleteTable(ctx context.Context, namespace, table string) error {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
|
||||
return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) LoadTable(ctx context.Context, namespace, table string) (*loadTableResponse, error) {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
|
||||
var resp loadTableResponse
|
||||
headers := map[string]string{
|
||||
"X-Iceberg-Access-Delegation": "vended-credentials",
|
||||
}
|
||||
if err := c.http.doJSONWithHeaders(ctx, http.MethodGet, path, nil, &resp, headers); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) LoadCredentials(ctx context.Context, namespace, table string) (*loadCredentialsResponse, error) {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s/credentials", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
|
||||
var resp loadCredentialsResponse
|
||||
if err := c.http.doJSON(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoint, fallbackRegion string) (aws.Credentials, string, string, bool, error) {
|
||||
credentialConfig, err := selectStorageConfig(load, targetPrefix)
|
||||
if err != nil {
|
||||
return aws.Credentials{}, "", "", false, err
|
||||
}
|
||||
|
||||
lookupConfig := func(key string) string {
|
||||
if load != nil && load.Config != nil {
|
||||
if val, ok := load.Config[key]; ok && strings.TrimSpace(val) != "" {
|
||||
return strings.TrimSpace(val)
|
||||
}
|
||||
}
|
||||
if val, ok := credentialConfig[key]; ok && strings.TrimSpace(val) != "" {
|
||||
return strings.TrimSpace(val)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
accessKey := strings.TrimSpace(credentialConfig["s3.access-key-id"])
|
||||
secretKey := strings.TrimSpace(credentialConfig["s3.secret-access-key"])
|
||||
sessionToken := strings.TrimSpace(credentialConfig["s3.session-token"])
|
||||
if accessKey == "" || secretKey == "" {
|
||||
return aws.Credentials{}, "", "", false, fmt.Errorf("missing s3.access-key-id or s3.secret-access-key in selected storage credential")
|
||||
}
|
||||
|
||||
endpoint := lookupConfig("s3.endpoint")
|
||||
if endpoint == "" {
|
||||
endpoint = fallbackEndpoint
|
||||
}
|
||||
if endpoint != "" && !strings.HasPrefix(endpoint, "http") {
|
||||
endpoint = "http://" + endpoint
|
||||
}
|
||||
|
||||
region := lookupConfig("client.region")
|
||||
if region == "" {
|
||||
region = fallbackRegion
|
||||
}
|
||||
|
||||
pathStyle := true
|
||||
if value := lookupConfig("s3.path-style-access"); value != "" {
|
||||
pathStyle = strings.EqualFold(value, "true")
|
||||
}
|
||||
|
||||
return aws.Credentials{
|
||||
AccessKeyID: accessKey,
|
||||
SecretAccessKey: secretKey,
|
||||
SessionToken: sessionToken,
|
||||
Source: "polaris-vended",
|
||||
}, endpoint, region, pathStyle, nil
|
||||
}
|
||||
|
||||
func selectStorageConfig(load *loadTableResponse, targetPrefix string) (map[string]string, error) {
|
||||
if load == nil {
|
||||
return nil, fmt.Errorf("load table response is nil")
|
||||
}
|
||||
|
||||
switch len(load.StorageCredentials) {
|
||||
case 0:
|
||||
if load.Config == nil {
|
||||
return nil, fmt.Errorf("polaris returned no storage credentials or config")
|
||||
}
|
||||
return load.Config, nil
|
||||
case 1:
|
||||
cred := load.StorageCredentials[0]
|
||||
if cred.Config == nil {
|
||||
return nil, fmt.Errorf("storage credential for prefix %s returned nil config", cred.Prefix)
|
||||
}
|
||||
return cred.Config, nil
|
||||
default:
|
||||
if targetPrefix == "" {
|
||||
return nil, fmt.Errorf("multiple storage credentials (%d) returned but no target prefix provided", len(load.StorageCredentials))
|
||||
}
|
||||
normalizedTarget := normalizePrefix(targetPrefix)
|
||||
if normalizedTarget == "" {
|
||||
return nil, fmt.Errorf("target prefix %q normalized to empty string", targetPrefix)
|
||||
}
|
||||
var bestConfig map[string]string
|
||||
bestLen := -1
|
||||
for _, cred := range load.StorageCredentials {
|
||||
if cred.Config == nil {
|
||||
continue
|
||||
}
|
||||
prefix := normalizePrefix(cred.Prefix)
|
||||
if prefix == "" {
|
||||
if bestLen < 0 {
|
||||
bestLen = 0
|
||||
bestConfig = cred.Config
|
||||
}
|
||||
continue
|
||||
}
|
||||
if normalizedTarget == prefix || strings.HasPrefix(normalizedTarget, prefix+"/") {
|
||||
if len(prefix) > bestLen {
|
||||
bestLen = len(prefix)
|
||||
bestConfig = cred.Config
|
||||
}
|
||||
}
|
||||
}
|
||||
if bestConfig != nil {
|
||||
return bestConfig, nil
|
||||
}
|
||||
return nil, fmt.Errorf("none of the %d storage credentials matched prefix %s", len(load.StorageCredentials), targetPrefix)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePrefix(prefix string) string {
|
||||
p := strings.TrimSpace(prefix)
|
||||
p = strings.TrimSuffix(p, "/")
|
||||
return p
|
||||
}
|
||||
|
||||
func s3URIToKeyPrefix(uri, bucket string) (string, error) {
|
||||
prefix := "s3://" + bucket + "/"
|
||||
if !strings.HasPrefix(uri, prefix) {
|
||||
return "", fmt.Errorf("uri %q does not match bucket %q", uri, bucket)
|
||||
}
|
||||
keyPrefix := strings.TrimPrefix(uri, prefix)
|
||||
keyPrefix = strings.TrimPrefix(keyPrefix, "/")
|
||||
if keyPrefix == "" {
|
||||
return "", fmt.Errorf("empty key prefix in uri %q", uri)
|
||||
}
|
||||
return keyPrefix, nil
|
||||
}
|
||||
@@ -23,6 +23,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("create status request: %v", err)
|
||||
}
|
||||
statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1")
|
||||
|
||||
statusResp := framework.DoRequest(t, client, statusReq)
|
||||
statusBody := framework.ReadAllAndClose(t, statusResp)
|
||||
|
||||
@@ -32,8 +34,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
|
||||
if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
|
||||
t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got)
|
||||
}
|
||||
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
|
||||
t.Fatal("expected server-generated request id in response header")
|
||||
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" {
|
||||
t.Fatalf("expected echoed request id, got %q", got)
|
||||
}
|
||||
|
||||
var payload map[string]interface{}
|
||||
@@ -47,6 +49,7 @@ func TestAdminStatusAndHealthz(t *testing.T) {
|
||||
}
|
||||
|
||||
healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz")
|
||||
healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2")
|
||||
healthResp := framework.DoRequest(t, client, healthReq)
|
||||
_ = framework.ReadAllAndClose(t, healthResp)
|
||||
if healthResp.StatusCode != http.StatusOK {
|
||||
@@ -55,8 +58,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
|
||||
if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
|
||||
t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got)
|
||||
}
|
||||
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
|
||||
t.Fatal("expected /healthz server-generated request id in response header")
|
||||
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" {
|
||||
t.Fatalf("expected /healthz echoed request id, got %q", got)
|
||||
}
|
||||
|
||||
uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
|
||||
|
||||
@@ -80,11 +80,6 @@ type AccessKeyInfo struct {
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type CreateAccessKeyRequest struct {
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
}
|
||||
|
||||
type UpdateAccessKeyStatusRequest struct {
|
||||
Status string `json:"status" binding:"required"`
|
||||
}
|
||||
@@ -95,7 +90,6 @@ type UserDetails struct {
|
||||
Actions []string `json:"actions"`
|
||||
PolicyNames []string `json:"policy_names"`
|
||||
AccessKeys []AccessKeyInfo `json:"access_keys"`
|
||||
Groups []string `json:"groups"`
|
||||
}
|
||||
|
||||
type FilerNode struct {
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
|
||||
)
|
||||
|
||||
const (
|
||||
adminLockName = cluster.AdminShellLockName
|
||||
adminLockClientName = "admin-plugin"
|
||||
)
|
||||
|
||||
// AdminLockManager coordinates exclusive admin locks with reference counting.
|
||||
// It is safe for concurrent use.
|
||||
type AdminLockManager struct {
|
||||
locker *exclusive_locks.ExclusiveLocker
|
||||
clientName string
|
||||
|
||||
mu sync.Mutex
|
||||
cond *sync.Cond
|
||||
acquiring bool
|
||||
holdCount int
|
||||
|
||||
lastAcquiredAt time.Time
|
||||
lastReleasedAt time.Time
|
||||
waitingSince time.Time
|
||||
waitingReason string
|
||||
currentReason string
|
||||
}
|
||||
|
||||
func NewAdminLockManager(masterClient *wdclient.MasterClient, clientName string) *AdminLockManager {
|
||||
if masterClient == nil {
|
||||
return nil
|
||||
}
|
||||
if clientName == "" {
|
||||
clientName = adminLockClientName
|
||||
}
|
||||
manager := &AdminLockManager{
|
||||
locker: exclusive_locks.NewExclusiveLocker(masterClient, adminLockName),
|
||||
clientName: clientName,
|
||||
}
|
||||
manager.cond = sync.NewCond(&manager.mu)
|
||||
return manager
|
||||
}
|
||||
|
||||
func (m *AdminLockManager) Acquire(reason string) (func(), error) {
|
||||
if m == nil || m.locker == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
if reason != "" {
|
||||
m.locker.SetMessage(reason)
|
||||
m.currentReason = reason
|
||||
}
|
||||
for m.acquiring {
|
||||
m.cond.Wait()
|
||||
}
|
||||
if m.holdCount == 0 {
|
||||
m.acquiring = true
|
||||
m.waitingSince = time.Now().UTC()
|
||||
m.waitingReason = reason
|
||||
m.mu.Unlock()
|
||||
m.locker.RequestLock(m.clientName)
|
||||
m.mu.Lock()
|
||||
m.acquiring = false
|
||||
m.holdCount = 1
|
||||
m.lastAcquiredAt = time.Now().UTC()
|
||||
m.waitingSince = time.Time{}
|
||||
m.waitingReason = ""
|
||||
m.cond.Broadcast()
|
||||
m.mu.Unlock()
|
||||
return m.Release, nil
|
||||
}
|
||||
m.holdCount++
|
||||
if reason != "" {
|
||||
m.currentReason = reason
|
||||
}
|
||||
m.mu.Unlock()
|
||||
return m.Release, nil
|
||||
}
|
||||
|
||||
func (m *AdminLockManager) Release() {
|
||||
if m == nil || m.locker == nil {
|
||||
return
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
if m.holdCount <= 0 {
|
||||
m.mu.Unlock()
|
||||
return
|
||||
}
|
||||
m.holdCount--
|
||||
shouldRelease := m.holdCount == 0
|
||||
m.mu.Unlock()
|
||||
|
||||
if shouldRelease {
|
||||
m.mu.Lock()
|
||||
m.lastReleasedAt = time.Now().UTC()
|
||||
m.currentReason = ""
|
||||
m.mu.Unlock()
|
||||
m.locker.ReleaseLock()
|
||||
}
|
||||
}
|
||||
|
||||
type LockStatus struct {
|
||||
Held bool `json:"held"`
|
||||
HoldCount int `json:"hold_count"`
|
||||
Acquiring bool `json:"acquiring"`
|
||||
Message string `json:"message,omitempty"`
|
||||
WaitingReason string `json:"waiting_reason,omitempty"`
|
||||
LastAcquiredAt *time.Time `json:"last_acquired_at,omitempty"`
|
||||
LastReleasedAt *time.Time `json:"last_released_at,omitempty"`
|
||||
WaitingSince *time.Time `json:"waiting_since,omitempty"`
|
||||
}
|
||||
|
||||
func (m *AdminLockManager) Status() LockStatus {
|
||||
if m == nil {
|
||||
return LockStatus{}
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
status := LockStatus{
|
||||
Held: m.holdCount > 0,
|
||||
HoldCount: m.holdCount,
|
||||
Acquiring: m.acquiring,
|
||||
Message: m.currentReason,
|
||||
WaitingReason: m.waitingReason,
|
||||
}
|
||||
if !m.lastAcquiredAt.IsZero() {
|
||||
at := m.lastAcquiredAt
|
||||
status.LastAcquiredAt = &at
|
||||
}
|
||||
if !m.lastReleasedAt.IsZero() {
|
||||
at := m.lastReleasedAt
|
||||
status.LastReleasedAt = &at
|
||||
}
|
||||
if !m.waitingSince.IsZero() {
|
||||
at := m.waitingSince
|
||||
status.WaitingSince = &at
|
||||
}
|
||||
return status
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
|
||||
)
|
||||
|
||||
const adminPresenceClientName = "admin-server"
|
||||
|
||||
type adminPresenceLock struct {
|
||||
locker *exclusive_locks.ExclusiveLocker
|
||||
stopCh chan struct{}
|
||||
}
|
||||
|
||||
func newAdminPresenceLock(masterClient *wdclient.MasterClient) *adminPresenceLock {
|
||||
if masterClient == nil {
|
||||
return nil
|
||||
}
|
||||
return &adminPresenceLock{
|
||||
locker: exclusive_locks.NewExclusiveLocker(masterClient, cluster.AdminServerPresenceLockName),
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *adminPresenceLock) Start() {
|
||||
if l == nil || l.locker == nil {
|
||||
return
|
||||
}
|
||||
l.locker.SetMessage("admin server connected")
|
||||
go func() {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if !l.locker.IsLocked() {
|
||||
l.locker.RequestLock(adminPresenceClientName)
|
||||
}
|
||||
select {
|
||||
case <-l.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (l *adminPresenceLock) Stop() {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-l.stopCh:
|
||||
default:
|
||||
close(l.stopCh)
|
||||
}
|
||||
if l.locker != nil {
|
||||
l.locker.ReleaseLock()
|
||||
}
|
||||
}
|
||||
+35
-305
@@ -2,18 +2,10 @@ package dash
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
|
||||
adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
clustermaintenance "github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
@@ -27,6 +19,10 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"google.golang.org/grpc"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
@@ -101,9 +97,6 @@ type AdminServer struct {
|
||||
// Maintenance system
|
||||
maintenanceManager *maintenance.MaintenanceManager
|
||||
plugin *adminplugin.Plugin
|
||||
pluginLock *AdminLockManager
|
||||
adminPresenceLock *adminPresenceLock
|
||||
expireJobHandler func(jobID string, reason string) (*adminplugin.TrackedJob, bool, error)
|
||||
|
||||
// Topic retention purger
|
||||
topicRetentionPurger *TopicRetentionPurger
|
||||
@@ -140,12 +133,6 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
|
||||
ctx := context.Background()
|
||||
go masterClient.KeepConnectedToMaster(ctx)
|
||||
|
||||
lockManager := NewAdminLockManager(masterClient, adminLockClientName)
|
||||
presenceLock := newAdminPresenceLock(masterClient)
|
||||
if presenceLock != nil {
|
||||
presenceLock.Start()
|
||||
}
|
||||
|
||||
server := &AdminServer{
|
||||
masterClient: masterClient,
|
||||
templateFS: templateFS,
|
||||
@@ -157,8 +144,6 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
|
||||
collectionStatsCacheThreshold: defaultStatsCacheTimeout,
|
||||
s3TablesManager: newS3TablesManager(),
|
||||
icebergPort: icebergPort,
|
||||
pluginLock: lockManager,
|
||||
adminPresenceLock: presenceLock,
|
||||
}
|
||||
|
||||
// Initialize topic retention purger
|
||||
@@ -237,19 +222,20 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
|
||||
}()
|
||||
}
|
||||
|
||||
pluginOpts := adminplugin.Options{
|
||||
plugin, err := adminplugin.New(adminplugin.Options{
|
||||
DataDir: dataDir,
|
||||
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
|
||||
return server.buildDefaultPluginClusterContext(), nil
|
||||
},
|
||||
LockManager: lockManager,
|
||||
ConfigDefaultsProvider: server.enrichConfigDefaults,
|
||||
}
|
||||
plugin, err := adminplugin.New(pluginOpts)
|
||||
})
|
||||
if err != nil && dataDir != "" {
|
||||
glog.Warningf("Failed to initialize plugin with dataDir=%q: %v. Falling back to in-memory plugin state.", dataDir, err)
|
||||
pluginOpts.DataDir = ""
|
||||
plugin, err = adminplugin.New(pluginOpts)
|
||||
plugin, err = adminplugin.New(adminplugin.Options{
|
||||
DataDir: "",
|
||||
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
|
||||
return server.buildDefaultPluginClusterContext(), nil
|
||||
},
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to initialize plugin: %v", err)
|
||||
@@ -273,89 +259,6 @@ func (s *AdminServer) loadTaskConfigurationsFromPersistence() {
|
||||
configUpdateRegistry.UpdateAllConfigs(s.configPersistence)
|
||||
}
|
||||
|
||||
// enrichConfigDefaults is called by the plugin when bootstrapping a job type's
|
||||
// default config from its descriptor. For admin_script, it fetches maintenance
|
||||
// scripts from the master and uses them as the script default.
|
||||
//
|
||||
// MIGRATION: This exists to help users migrate from master.toml [master.maintenance]
|
||||
// to the admin script plugin worker. Remove after March 2027.
|
||||
func (s *AdminServer) enrichConfigDefaults(cfg *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig {
|
||||
if cfg.JobType != "admin_script" {
|
||||
return cfg
|
||||
}
|
||||
|
||||
var maintenanceScripts string
|
||||
var sleepMinutes uint32
|
||||
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
maintenanceScripts = resp.MaintenanceScripts
|
||||
sleepMinutes = resp.MaintenanceSleepMinutes
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("Could not fetch master configuration for admin_script defaults: %v", err)
|
||||
return cfg
|
||||
}
|
||||
|
||||
script := cleanMaintenanceScript(maintenanceScripts)
|
||||
if script == "" {
|
||||
return cfg
|
||||
}
|
||||
|
||||
interval := int64(sleepMinutes)
|
||||
if interval <= 0 {
|
||||
interval = clustermaintenance.DefaultMaintenanceSleepMinutes
|
||||
}
|
||||
|
||||
glog.V(0).Infof("Enriching admin_script defaults from master maintenance scripts (interval=%dm)", interval)
|
||||
|
||||
if cfg.AdminConfigValues == nil {
|
||||
cfg.AdminConfigValues = make(map[string]*plugin_pb.ConfigValue)
|
||||
}
|
||||
cfg.AdminConfigValues["script"] = &plugin_pb.ConfigValue{
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: script},
|
||||
}
|
||||
cfg.AdminConfigValues["run_interval_minutes"] = &plugin_pb.ConfigValue{
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: interval},
|
||||
}
|
||||
cfg.UpdatedBy = "master_migration"
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
// cleanMaintenanceScript strips lock/unlock commands and normalizes a
|
||||
// maintenance script string for use with the admin script plugin worker.
|
||||
//
|
||||
// MIGRATION: Used by enrichConfigDefaults. Remove after March 2027.
|
||||
func cleanMaintenanceScript(script string) string {
|
||||
script = strings.ReplaceAll(script, "\r\n", "\n")
|
||||
var lines []string
|
||||
for _, line := range strings.Split(script, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
|
||||
continue
|
||||
}
|
||||
// Strip inline comments (e.g., "lock # migration note")
|
||||
if idx := strings.Index(trimmed, "#"); idx >= 0 {
|
||||
trimmed = strings.TrimSpace(trimmed[:idx])
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
firstToken := strings.ToLower(strings.Fields(trimmed)[0])
|
||||
if firstToken == "lock" || firstToken == "unlock" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, trimmed)
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// GetCredentialManager returns the credential manager
|
||||
func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
|
||||
return s.credentialManager
|
||||
@@ -373,21 +276,8 @@ func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
|
||||
|
||||
// InvalidateCache method moved to cluster_topology.go
|
||||
|
||||
// GetS3BucketsData retrieves Object Store buckets with pagination and sorting
|
||||
func (s *AdminServer) GetS3BucketsData(page, pageSize int, sortBy, sortOrder string) (S3BucketsData, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 1 || pageSize > 1000 {
|
||||
pageSize = 100
|
||||
}
|
||||
if sortBy == "" {
|
||||
sortBy = "name"
|
||||
}
|
||||
if sortOrder == "" {
|
||||
sortOrder = "asc"
|
||||
}
|
||||
|
||||
// GetS3BucketsData retrieves all Object Store buckets and aggregates total storage metrics
|
||||
func (s *AdminServer) GetS3BucketsData() (S3BucketsData, error) {
|
||||
buckets, err := s.GetS3Buckets()
|
||||
if err != nil {
|
||||
return S3BucketsData{}, err
|
||||
@@ -398,97 +288,14 @@ func (s *AdminServer) GetS3BucketsData(page, pageSize int, sortBy, sortOrder str
|
||||
totalSize += bucket.PhysicalSize
|
||||
}
|
||||
|
||||
totalBuckets := len(buckets)
|
||||
|
||||
// Sort buckets
|
||||
s.sortBuckets(buckets, sortBy, sortOrder)
|
||||
|
||||
// Calculate pagination
|
||||
totalPages := (totalBuckets + pageSize - 1) / pageSize
|
||||
if totalPages == 0 {
|
||||
totalPages = 1
|
||||
}
|
||||
if page > totalPages {
|
||||
page = totalPages
|
||||
}
|
||||
|
||||
startIndex := (page - 1) * pageSize
|
||||
endIndex := startIndex + pageSize
|
||||
if startIndex >= totalBuckets {
|
||||
buckets = []S3Bucket{}
|
||||
} else {
|
||||
if endIndex > totalBuckets {
|
||||
endIndex = totalBuckets
|
||||
}
|
||||
buckets = buckets[startIndex:endIndex]
|
||||
}
|
||||
|
||||
return S3BucketsData{
|
||||
Buckets: buckets,
|
||||
TotalBuckets: totalBuckets,
|
||||
TotalBuckets: len(buckets),
|
||||
TotalSize: totalSize,
|
||||
LastUpdated: time.Now(),
|
||||
CurrentPage: page,
|
||||
TotalPages: totalPages,
|
||||
PageSize: pageSize,
|
||||
SortBy: sortBy,
|
||||
SortOrder: sortOrder,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// sortBuckets sorts the bucket slice in place by the given field and order
|
||||
func (s *AdminServer) sortBuckets(buckets []S3Bucket, sortBy, sortOrder string) {
|
||||
desc := sortOrder == "desc"
|
||||
sort.Slice(buckets, func(i, j int) bool {
|
||||
a, b := buckets[i], buckets[j]
|
||||
switch sortBy {
|
||||
case "owner":
|
||||
if a.Owner != b.Owner {
|
||||
if desc {
|
||||
return a.Owner > b.Owner
|
||||
}
|
||||
return a.Owner < b.Owner
|
||||
}
|
||||
case "created":
|
||||
if !a.CreatedAt.Equal(b.CreatedAt) {
|
||||
if desc {
|
||||
return a.CreatedAt.After(b.CreatedAt)
|
||||
}
|
||||
return a.CreatedAt.Before(b.CreatedAt)
|
||||
}
|
||||
case "objects":
|
||||
if a.ObjectCount != b.ObjectCount {
|
||||
if desc {
|
||||
return a.ObjectCount > b.ObjectCount
|
||||
}
|
||||
return a.ObjectCount < b.ObjectCount
|
||||
}
|
||||
case "logical_size":
|
||||
if a.LogicalSize != b.LogicalSize {
|
||||
if desc {
|
||||
return a.LogicalSize > b.LogicalSize
|
||||
}
|
||||
return a.LogicalSize < b.LogicalSize
|
||||
}
|
||||
case "physical_size":
|
||||
if a.PhysicalSize != b.PhysicalSize {
|
||||
if desc {
|
||||
return a.PhysicalSize > b.PhysicalSize
|
||||
}
|
||||
return a.PhysicalSize < b.PhysicalSize
|
||||
}
|
||||
}
|
||||
// Tie-breaker: sort by name (also the default/primary for sortBy=="name")
|
||||
if a.Name != b.Name {
|
||||
if desc {
|
||||
return a.Name > b.Name
|
||||
}
|
||||
return a.Name < b.Name
|
||||
}
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
// GetS3Buckets retrieves all Object Store buckets from the filer and collects size/object data from collections
|
||||
func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
var buckets []S3Bucket
|
||||
@@ -504,48 +311,28 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
|
||||
// Now list buckets from the filer and match with collection data
|
||||
err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Paginate through all buckets in the buckets directory
|
||||
const listPageSize = 1000
|
||||
startFrom := ""
|
||||
var snapshotTsNs int64
|
||||
// List buckets by looking at the buckets directory
|
||||
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
|
||||
Directory: filerConfig.BucketsPath,
|
||||
Prefix: "",
|
||||
StartFromFileName: "",
|
||||
InclusiveStartFrom: false,
|
||||
Limit: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
|
||||
Directory: filerConfig.BucketsPath,
|
||||
Prefix: "",
|
||||
StartFromFileName: startFrom,
|
||||
InclusiveStartFrom: false,
|
||||
Limit: listPageSize,
|
||||
SnapshotTsNs: snapshotTsNs,
|
||||
})
|
||||
resp, err := stream.Recv()
|
||||
if err != nil {
|
||||
if err.Error() == "EOF" {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
pageCount := 0
|
||||
lastName := ""
|
||||
for {
|
||||
resp, err := stream.Recv()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if snapshotTsNs == 0 && resp.SnapshotTsNs != 0 {
|
||||
snapshotTsNs = resp.SnapshotTsNs
|
||||
}
|
||||
|
||||
if resp.Entry == nil {
|
||||
continue
|
||||
}
|
||||
lastName = resp.Entry.Name
|
||||
pageCount++
|
||||
|
||||
if !resp.Entry.IsDirectory {
|
||||
continue
|
||||
}
|
||||
|
||||
if resp.Entry.IsDirectory {
|
||||
bucketName := resp.Entry.Name
|
||||
if strings.HasPrefix(bucketName, ".") {
|
||||
// Skip internal/system directories from Object Store bucket listing.
|
||||
@@ -598,18 +385,13 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
}
|
||||
}
|
||||
|
||||
var createdAt, lastModified time.Time
|
||||
if resp.Entry.Attributes != nil {
|
||||
createdAt = time.Unix(resp.Entry.Attributes.Crtime, 0)
|
||||
lastModified = time.Unix(resp.Entry.Attributes.Mtime, 0)
|
||||
}
|
||||
bucket := S3Bucket{
|
||||
Name: bucketName,
|
||||
CreatedAt: createdAt,
|
||||
CreatedAt: time.Unix(resp.Entry.Attributes.Crtime, 0),
|
||||
LogicalSize: logicalSize,
|
||||
PhysicalSize: physicalSize,
|
||||
ObjectCount: objectCount,
|
||||
LastModified: lastModified,
|
||||
LastModified: time.Unix(resp.Entry.Attributes.Mtime, 0),
|
||||
Quota: quota,
|
||||
QuotaEnabled: quotaEnabled,
|
||||
VersioningStatus: versioningStatus,
|
||||
@@ -620,12 +402,6 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
}
|
||||
buckets = append(buckets, bucket)
|
||||
}
|
||||
|
||||
// If we received fewer entries than the page size, we've listed everything
|
||||
if pageCount < listPageSize {
|
||||
break
|
||||
}
|
||||
startFrom = lastName
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -1112,13 +888,6 @@ func (s *AdminServer) GetPlugin() *adminplugin.Plugin {
|
||||
return s.plugin
|
||||
}
|
||||
|
||||
func (s *AdminServer) acquirePluginLock(reason string) (func(), error) {
|
||||
if s == nil || s.pluginLock == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
return s.pluginLock.Acquire(reason)
|
||||
}
|
||||
|
||||
// RequestPluginJobTypeDescriptor asks one worker for job type schema and returns the descriptor.
|
||||
func (s *AdminServer) RequestPluginJobTypeDescriptor(ctx context.Context, jobType string, forceRefresh bool) (*plugin_pb.JobTypeDescriptor, error) {
|
||||
if s.plugin == nil {
|
||||
@@ -1161,13 +930,6 @@ func (s *AdminServer) RunPluginDetection(
|
||||
if s.plugin == nil {
|
||||
return nil, fmt.Errorf("plugin is not enabled")
|
||||
}
|
||||
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if releaseLock != nil {
|
||||
defer releaseLock()
|
||||
}
|
||||
return s.plugin.RunDetection(ctx, jobType, clusterContext, maxResults)
|
||||
}
|
||||
|
||||
@@ -1193,13 +955,6 @@ func (s *AdminServer) RunPluginDetectionWithReport(
|
||||
if s.plugin == nil {
|
||||
return nil, fmt.Errorf("plugin is not enabled")
|
||||
}
|
||||
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if releaseLock != nil {
|
||||
defer releaseLock()
|
||||
}
|
||||
return s.plugin.RunDetectionWithReport(ctx, jobType, clusterContext, maxResults)
|
||||
}
|
||||
|
||||
@@ -1213,17 +968,6 @@ func (s *AdminServer) ExecutePluginJob(
|
||||
if s.plugin == nil {
|
||||
return nil, fmt.Errorf("plugin is not enabled")
|
||||
}
|
||||
jobType := ""
|
||||
if job != nil {
|
||||
jobType = strings.TrimSpace(job.JobType)
|
||||
}
|
||||
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin execution %s", jobType))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if releaseLock != nil {
|
||||
defer releaseLock()
|
||||
}
|
||||
return s.plugin.ExecuteJob(ctx, job, clusterContext, attempt)
|
||||
}
|
||||
|
||||
@@ -1236,7 +980,7 @@ func (s *AdminServer) GetPluginRunHistory(jobType string) (*adminplugin.JobTypeR
|
||||
}
|
||||
|
||||
// ListPluginJobTypes returns known plugin job types from connected worker registry and persisted data.
|
||||
func (s *AdminServer) ListPluginJobTypes() ([]adminplugin.JobTypeInfo, error) {
|
||||
func (s *AdminServer) ListPluginJobTypes() ([]string, error) {
|
||||
if s.plugin == nil {
|
||||
return nil, fmt.Errorf("plugin is not enabled")
|
||||
}
|
||||
@@ -1275,17 +1019,6 @@ func (s *AdminServer) GetPluginJobDetail(jobID string, activityLimit, relatedLim
|
||||
return s.plugin.BuildJobDetail(jobID, activityLimit, relatedLimit)
|
||||
}
|
||||
|
||||
// ExpirePluginJob marks an active plugin job as failed so it no longer blocks scheduling.
|
||||
func (s *AdminServer) ExpirePluginJob(jobID, reason string) (*adminplugin.TrackedJob, bool, error) {
|
||||
if handler := s.expireJobHandler; handler != nil {
|
||||
return handler(jobID, reason)
|
||||
}
|
||||
if s.plugin == nil {
|
||||
return nil, false, fmt.Errorf("plugin is not enabled")
|
||||
}
|
||||
return s.plugin.ExpireJob(jobID, reason)
|
||||
}
|
||||
|
||||
// ListPluginActivities returns plugin job activities for monitoring.
|
||||
func (s *AdminServer) ListPluginActivities(jobType string, limit int) []adminplugin.JobActivity {
|
||||
if s.plugin == nil {
|
||||
@@ -1502,9 +1235,6 @@ func (s *AdminServer) Shutdown() {
|
||||
|
||||
// Stop maintenance manager
|
||||
s.StopMaintenanceManager()
|
||||
if s.adminPresenceLock != nil {
|
||||
s.adminPresenceLock.Stop()
|
||||
}
|
||||
|
||||
if s.plugin != nil {
|
||||
s.plugin.Shutdown()
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
// MIGRATION: Tests for enrichConfigDefaults helpers. Remove after March 2027.
|
||||
package dash
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCleanMaintenanceScript(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
input: "",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "only lock unlock",
|
||||
input: " lock\n unlock\n",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "strips lock and unlock",
|
||||
input: " lock\n ec.balance -apply\n volume.fix.replication -apply\n unlock\n",
|
||||
expected: "ec.balance -apply\nvolume.fix.replication -apply",
|
||||
},
|
||||
{
|
||||
name: "case insensitive lock",
|
||||
input: "Lock\nec.balance -apply\nUNLOCK",
|
||||
expected: "ec.balance -apply",
|
||||
},
|
||||
{
|
||||
name: "preserves comments removal",
|
||||
input: "lock\n# a comment\nec.balance -apply\nunlock",
|
||||
expected: "ec.balance -apply",
|
||||
},
|
||||
{
|
||||
name: "no lock unlock present",
|
||||
input: "ec.balance -apply\nvolume.fix.replication -apply",
|
||||
expected: "ec.balance -apply\nvolume.fix.replication -apply",
|
||||
},
|
||||
{
|
||||
name: "windows line endings",
|
||||
input: "lock\r\nec.balance -apply\r\nunlock\r\n",
|
||||
expected: "ec.balance -apply",
|
||||
},
|
||||
{
|
||||
name: "lock with inline comment",
|
||||
input: "lock # migration\nec.balance -apply\nunlock # done",
|
||||
expected: "ec.balance -apply",
|
||||
},
|
||||
{
|
||||
name: "command with inline comment preserved",
|
||||
input: "lock\nec.balance -apply # rebalance shards\nunlock",
|
||||
expected: "ec.balance -apply",
|
||||
},
|
||||
{
|
||||
name: "only inline comment after stripping",
|
||||
input: "# full line comment\n # indented comment\n",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "typical master default",
|
||||
input: "\n lock\n ec.encode -fullPercent=95 -quietFor=1h\n ec.rebuild -apply\n ec.balance -apply\n fs.log.purge -daysAgo=7\n volume.deleteEmpty -quietFor=24h -apply\n volume.balance -apply\n volume.fix.replication -apply\n s3.clean.uploads -timeAgo=24h\n unlock\n",
|
||||
expected: "ec.encode -fullPercent=95 -quietFor=1h\nec.rebuild -apply\nec.balance -apply\nfs.log.purge -daysAgo=7\nvolume.deleteEmpty -quietFor=24h -apply\nvolume.balance -apply\nvolume.fix.replication -apply\ns3.clean.uploads -timeAgo=24h",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := cleanMaintenanceScript(tt.input)
|
||||
if got != tt.expected {
|
||||
t.Errorf("cleanMaintenanceScript(%q) = %q, want %q", tt.input, got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -26,15 +26,6 @@ type S3BucketsData struct {
|
||||
TotalBuckets int `json:"total_buckets"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
|
||||
// Pagination
|
||||
CurrentPage int `json:"current_page"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
PageSize int `json:"page_size"`
|
||||
|
||||
// Sorting
|
||||
SortBy string `json:"sort_by"`
|
||||
SortOrder string `json:"sort_order"`
|
||||
}
|
||||
|
||||
type CreateBucketRequest struct {
|
||||
@@ -57,7 +48,7 @@ type CreateBucketRequest struct {
|
||||
func (s *AdminServer) ShowS3Buckets(w http.ResponseWriter, r *http.Request) {
|
||||
username := UsernameFromContext(r.Context())
|
||||
|
||||
data, err := s.GetS3BucketsData(1, 100, "name", "asc")
|
||||
data, err := s.GetS3BucketsData()
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get Object Store buckets: "+err.Error())
|
||||
return
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user