mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-06 14:45:51 +00:00
Compare commits
91
Commits
feature/sw-block
...
4.16
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3d9f7f6f81 | ||
|
|
d89a78d9e3 | ||
|
|
00000ec006 | ||
|
|
1bd7a98a4a | ||
|
|
8ad58e7002 | ||
|
|
f220328ae4 | ||
|
|
cf3693651c | ||
|
|
5f85bf5e8a | ||
|
|
b991acf634 | ||
|
|
02d3e3195c | ||
|
|
470075dd90 | ||
|
|
f8b7357350 | ||
|
|
e1c4faba38 | ||
|
|
6c7fe87a72 | ||
|
|
b3d32fe73b | ||
|
|
f439c84d01 | ||
|
|
89f1096c0e | ||
|
|
6dab90472b | ||
|
|
a00d38d8d4 | ||
|
|
f8d783f80e | ||
|
|
120d38176f | ||
|
|
55bce53953 | ||
|
|
992db11d2b | ||
|
|
115dcb5ada | ||
|
|
7be2d1ecfb | ||
|
|
1272612bbd | ||
|
|
e568d85a5c | ||
|
|
f79ba1eb37 | ||
|
|
b132232895 | ||
|
|
d765ff50e6 | ||
|
|
bff084ff6a | ||
|
|
78a3441b30 | ||
|
|
2ec0a67ee3 | ||
|
|
0647f66bb5 | ||
|
|
ba66411337 | ||
|
|
7808b301ef | ||
|
|
fa7da0f57e | ||
|
|
961c270aba | ||
|
|
e25558e4d8 | ||
|
|
587c24ec89 | ||
|
|
f249fb7e63 | ||
|
|
72c2c7ef8b | ||
|
|
d89eb8267f | ||
|
|
3f946fc0c0 | ||
|
|
af4c3fcb31 | ||
|
|
bfc430afbd | ||
|
|
540fc97e00 | ||
|
|
14cd0f53ba | ||
|
|
f9311a3422 | ||
|
|
338be16254 | ||
|
|
1b6e96614d | ||
|
|
4eb45ecc5e | ||
|
|
1f3df6e9ef | ||
|
|
fcd5de9710 | ||
|
|
b6f6f0187e | ||
|
|
230ae9c24e | ||
|
|
b3f7472fd3 | ||
|
|
b3620c7e14 | ||
|
|
7799804200 | ||
|
|
c19f88eef1 | ||
|
|
88e8342e44 | ||
|
|
df5e8210df | ||
|
|
10a30a83e1 | ||
|
|
9e26d6f5dd | ||
|
|
e475cbfef8 | ||
|
|
70ed9c2a55 | ||
|
|
45ce18266a | ||
|
|
18ccc9b773 | ||
|
|
e1e5b4a8a6 | ||
|
|
16f2269a33 | ||
|
|
1a3e3100d0 | ||
|
|
a61a2affe3 | ||
|
|
3db05f59f0 | ||
|
|
2644816692 | ||
|
|
fb944f0071 | ||
|
|
479da50433 | ||
|
|
f7909b8ebd | ||
|
|
2a3ecee28b | ||
|
|
f9cf3f3791 | ||
|
|
5d0667221b | ||
|
|
74593f7065 | ||
|
|
340339f678 | ||
|
|
2fc47a48ec | ||
|
|
623450a0d4 | ||
|
|
f5c35240be | ||
|
|
c5d5b517f6 | ||
|
|
2dd3944819 | ||
|
|
7354fa87f1 | ||
|
|
e8946e59ca | ||
|
|
b9e560dcf1 | ||
|
|
4f647e1036 |
@@ -32,7 +32,7 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
-
|
||||
name: Create BuildKit config
|
||||
run: |
|
||||
@@ -42,28 +42,28 @@ jobs:
|
||||
EOF
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -126,14 +126,14 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
- name: Set up QEMU
|
||||
if: matrix.platform != 'amd64'
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
cat > /tmp/buildkitd.toml <<EOF
|
||||
@@ -106,25 +106,25 @@ jobs:
|
||||
mirrors = ["https://mirror.gcr.io"]
|
||||
EOF
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Build ${{ matrix.platform }} ${{ matrix.variant }}
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
@@ -180,12 +180,12 @@ jobs:
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag || 'latest' }},suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
- name: Login to GHCR
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -35,14 +35,14 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
fi
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -60,16 +60,16 @@ jobs:
|
||||
build_args: TAGS=5BytesOffset
|
||||
tag_suffix: _large_disk
|
||||
|
||||
# Full tags - amd64 only
|
||||
# Full tags - multi-arch
|
||||
- variant: full
|
||||
platforms: linux/amd64
|
||||
platforms: linux/amd64,linux/arm64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _full
|
||||
|
||||
# Large disk + full tags - amd64 only
|
||||
|
||||
# Large disk + full tags - multi-arch
|
||||
- variant: large_disk_full
|
||||
platforms: linux/amd64
|
||||
platforms: linux/amd64,linux/arm64
|
||||
dockerfile: ./docker/Dockerfile.go_build
|
||||
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
|
||||
tag_suffix: _large_disk_full
|
||||
@@ -117,7 +117,7 @@ jobs:
|
||||
|
||||
- name: Set up QEMU
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && contains(matrix.platforms, 'arm')
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Create BuildKit config
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
@@ -129,20 +129,20 @@ jobs:
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
with:
|
||||
buildkitd-config: /tmp/buildkitd.toml
|
||||
|
||||
- name: Login to Docker Hub
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: (github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant) && github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
|
||||
- name: Build and push ${{ matrix.variant }}
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
with:
|
||||
@@ -198,14 +198,14 @@ jobs:
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GHCR
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
|
||||
@@ -82,19 +82,19 @@ jobs:
|
||||
echo "seaweedfs_ref=$seaweed" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v1
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v1
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v1
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v1
|
||||
uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push image
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v2
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: true
|
||||
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.24'
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build Telemetry Server
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.deploy
|
||||
|
||||
@@ -11,4 +11,4 @@ jobs:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@05fe4576374b728f0c523d6a13d64c25081e0803
|
||||
uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
@@ -135,7 +135,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: output-logs
|
||||
path: docker/output.log
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
@@ -52,7 +52,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: ec-integration-test-logs
|
||||
path: |
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.24'
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: ec-test-logs
|
||||
path: test/erasure_coding/admin_dockertest/tmp/logs/
|
||||
|
||||
@@ -22,7 +22,6 @@ permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
TEST_TIMEOUT: '45m'
|
||||
|
||||
jobs:
|
||||
@@ -35,10 +34,10 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go ${{ env.GO_VERSION }}
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ${{ env.GO_VERSION }}
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Install FUSE and dependencies
|
||||
run: |
|
||||
@@ -183,7 +182,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: fuse-integration-test-results
|
||||
path: |
|
||||
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: test-reports-java-${{ matrix.java }}
|
||||
path: |
|
||||
|
||||
@@ -26,14 +26,14 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
id: go
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
@@ -132,7 +132,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -311,7 +311,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -473,7 +473,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -631,7 +631,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
cache-dependency-path: |
|
||||
**/go.sum
|
||||
@@ -789,7 +789,7 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: metadata-subscribe-test-logs
|
||||
path: |
|
||||
|
||||
@@ -25,14 +25,14 @@ jobs:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
id: go
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: postgres-logs
|
||||
path: test/postgres/postgres-output.log
|
||||
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: integration-test-logs
|
||||
path: test/s3/normal/*.log
|
||||
|
||||
@@ -77,7 +77,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-filer-group-test-logs
|
||||
path: test/s3/filer_group/weed-test*.log
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-compatibility-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-cors-compatibility-logs
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -239,7 +239,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-retention-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -306,7 +306,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-cors-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -355,7 +355,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-retention-worm-logs
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -422,7 +422,7 @@ jobs:
|
||||
|
||||
- name: Upload stress test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-versioning-stress-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -478,7 +478,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-tagging-test-logs
|
||||
path: test/s3/tagging/weed-test*.log
|
||||
@@ -531,7 +531,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-remote-cache-test-logs
|
||||
path: |
|
||||
|
||||
@@ -5,6 +5,8 @@ on:
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/credential/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
push:
|
||||
@@ -12,6 +14,8 @@ on:
|
||||
paths:
|
||||
- 'weed/iam/**'
|
||||
- 'weed/s3api/**'
|
||||
- 'weed/credential/**'
|
||||
- 'weed/pb/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-iam-tests.yml'
|
||||
|
||||
@@ -65,7 +69,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: iam-unit-test-results
|
||||
path: |
|
||||
@@ -80,7 +84,7 @@ jobs:
|
||||
timeout-minutes: 25
|
||||
strategy:
|
||||
matrix:
|
||||
test-type: ["basic", "advanced", "policy-enforcement"]
|
||||
test-type: ["basic", "advanced", "policy-enforcement", "group"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -117,7 +121,7 @@ jobs:
|
||||
"basic")
|
||||
echo "Running basic IAM functionality tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAM" ./...
|
||||
go test -v -timeout 15m -run "TestS3IAMAuthentication|TestS3IAMBasicWorkflow|TestS3IAMTokenValidation|TestIAMUserManagement|TestIAMAccessKeyManagement|TestIAMPolicyManagement" ./...
|
||||
;;
|
||||
"advanced")
|
||||
echo "Running advanced IAM feature tests..."
|
||||
@@ -129,6 +133,11 @@ jobs:
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestS3IAMPolicyEnforcement|TestS3IAMBucketPolicy|TestS3IAMContextual" ./...
|
||||
;;
|
||||
"group")
|
||||
echo "Running IAM group management tests..."
|
||||
make clean setup start-services wait-for-services
|
||||
go test -v -timeout 15m -run "TestIAMGroup" ./...
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test type: ${{ matrix.test-type }}"
|
||||
exit 1
|
||||
@@ -162,7 +171,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-iam-integration-logs-${{ matrix.test-type }}
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -222,7 +231,7 @@ jobs:
|
||||
|
||||
- name: Upload distributed test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-iam-distributed-logs
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -274,7 +283,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-iam-performance-results
|
||||
path: |
|
||||
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-keycloak-test-logs
|
||||
path: |
|
||||
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: test-logs-python-${{ matrix.python-version }}
|
||||
path: |
|
||||
@@ -148,7 +148,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
cache: true
|
||||
|
||||
- name: Run Go unit tests
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: policy-unit-test-results
|
||||
path: |
|
||||
@@ -178,7 +178,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-policy-variables-test-logs
|
||||
path: /tmp/weed_policy_test_server.log
|
||||
@@ -299,7 +299,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
|
||||
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
|
||||
@@ -386,7 +386,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: trusted-proxy-test-logs
|
||||
path: /tmp/weed_proxy_test.log
|
||||
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Build SeaweedFS binary for Linux
|
||||
run: |
|
||||
|
||||
@@ -33,7 +33,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-spark-test-logs
|
||||
path: test/s3/spark/test-output.log
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -143,7 +143,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-compatibility-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-metadata-persistence-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -241,7 +241,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-copy-operations-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -290,7 +290,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-multipart-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -340,7 +340,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-sse-performance-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -389,7 +389,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-volume-encryption-logs
|
||||
path: /tmp/seaweedfs-sse-*.log
|
||||
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: s3-tables-test-logs
|
||||
path: test/s3tables/table-buckets/test-output.log
|
||||
@@ -122,7 +122,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog/test-output.log
|
||||
@@ -144,7 +144,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Trino image
|
||||
run: docker pull trinodb/trino:479
|
||||
@@ -188,12 +188,73 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: trino-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_trino/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
polaris-integration-tests:
|
||||
name: Polaris Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Run go mod tidy
|
||||
run: go mod tidy
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Pre-pull Polaris image
|
||||
run: docker pull apache/polaris:latest
|
||||
|
||||
- name: Run Polaris Integration Tests
|
||||
timeout-minutes: 25
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Polaris Tests ==="
|
||||
|
||||
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
|
||||
echo "Polaris integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables/polaris
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test|docker)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: polaris-test-logs
|
||||
path: test/s3tables/polaris/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
spark-iceberg-catalog-tests:
|
||||
name: Spark Iceberg Catalog Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
@@ -210,7 +271,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Spark image
|
||||
run: docker pull apache/spark:3.5.1
|
||||
@@ -254,7 +315,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: spark-iceberg-catalog-test-logs
|
||||
path: test/s3tables/catalog_spark/test-output.log
|
||||
@@ -276,7 +337,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull RisingWave image
|
||||
run: |
|
||||
@@ -322,7 +383,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: risingwave-catalog-test-logs
|
||||
path: test/s3tables/catalog_risingwave/test-output.log
|
||||
@@ -344,7 +405,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Python image
|
||||
run: docker pull python:3
|
||||
@@ -388,7 +449,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: sts-integration-test-logs
|
||||
path: test/s3tables/sts_integration/test-output.log
|
||||
@@ -410,7 +471,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Set up Docker
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Pre-pull Python image
|
||||
run: docker pull python:3
|
||||
@@ -457,7 +518,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: lakekeeper-integration-test-logs
|
||||
path: test/s3tables/lakekeeper/test-output.log
|
||||
|
||||
@@ -43,7 +43,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: '1.24'
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
@@ -125,7 +125,7 @@ jobs:
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: spark-test-results
|
||||
path: test/java/spark/target/surefire-reports/
|
||||
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: ^1.24
|
||||
go-version: ^1.25
|
||||
|
||||
- name: Build SeaweedFS
|
||||
run: |
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: seaweedfs-logs
|
||||
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
|
||||
|
||||
@@ -106,7 +106,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: tus-test-logs
|
||||
path: |
|
||||
|
||||
@@ -90,7 +90,7 @@ jobs:
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: volume-server-integration-test-logs
|
||||
path: /tmp/volume-server-it-logs/
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24 AS builder
|
||||
FROM golang:1.25 AS builder
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y build-essential wget ca-certificates && \
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24-alpine as builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
ARG BRANCH=${BRANCH:-master}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24 AS builder
|
||||
FROM golang:1.25 AS builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24 AS builder
|
||||
FROM golang:1.25 AS builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
|
||||
[master.maintenance]
|
||||
# periodically run these scripts are the same as running them from 'weed shell'
|
||||
# Scripts are skipped while an admin server is connected.
|
||||
scripts = """
|
||||
lock
|
||||
ec.encode -fullPercent=95 -quietFor=1h
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.24.9
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
@@ -26,7 +26,7 @@ require (
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.15.0
|
||||
github.com/go-redsync/redsync/v4 v4.16.0
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/golang/protobuf v1.5.4
|
||||
@@ -63,7 +63,7 @@ require (
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.2 // indirect
|
||||
github.com/prometheus/procfs v0.19.2
|
||||
github.com/prometheus/procfs v0.20.1
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
@@ -87,22 +87,22 @@ require (
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.6.7
|
||||
go.mongodb.org/mongo-driver v1.17.6
|
||||
go.mongodb.org/mongo-driver v1.17.9
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.44.0
|
||||
gocloud.dev/pubsub/natspubsub v0.44.0
|
||||
gocloud.dev v0.45.0
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.44.0
|
||||
golang.org/x/crypto v0.48.0
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
|
||||
golang.org/x/image v0.36.0
|
||||
golang.org/x/net v0.49.0
|
||||
golang.org/x/oauth2 v0.34.0
|
||||
golang.org/x/sys v0.41.0
|
||||
golang.org/x/oauth2 v0.35.0
|
||||
golang.org/x/sys v0.42.0
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
golang.org/x/tools v0.41.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.258.0
|
||||
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
|
||||
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 // indirect
|
||||
google.golang.org/grpc v1.78.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
@@ -124,13 +124,13 @@ require (
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.3
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.7
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.42.0
|
||||
github.com/getsentry/sentry-go v0.43.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.12
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
@@ -138,8 +138,7 @@ require (
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/lib/pq v1.11.1
|
||||
github.com/linkedin/goavro/v2 v2.14.1
|
||||
github.com/linkedin/goavro/v2 v2.15.0
|
||||
github.com/mattn/go-sqlite3 v1.14.34
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
@@ -151,7 +150,7 @@ require (
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1
|
||||
github.com/shirou/gopsutil/v4 v4.26.1
|
||||
github.com/shirou/gopsutil/v4 v4.26.2
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.1
|
||||
github.com/testcontainers/testcontainers-go v0.39.0
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
@@ -172,7 +171,7 @@ require (
|
||||
atomicgo.dev/keyboard v0.2.9 // indirect
|
||||
atomicgo.dev/schedule v0.1.0 // indirect
|
||||
cloud.google.com/go/longrunning v0.7.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
|
||||
@@ -237,6 +236,7 @@ require (
|
||||
github.com/klauspost/asmfmt v1.3.2 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lib/pq v1.11.1 // indirect
|
||||
github.com/lithammer/fuzzysearch v1.1.8 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
@@ -279,10 +279,10 @@ require (
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
|
||||
go.uber.org/mock v0.5.2 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
@@ -292,12 +292,12 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.24.0 // indirect
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go/auth v0.17.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.2 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||
filippo.io/edwards25519 v1.1.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
@@ -327,7 +327,7 @@ require (
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
@@ -336,12 +336,12 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
|
||||
github.com/aws/smithy-go v1.24.0
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6
|
||||
github.com/aws/smithy-go v1.24.2
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
@@ -352,7 +352,7 @@ require (
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/creasty/defaults v1.8.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
@@ -360,11 +360,11 @@ require (
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
|
||||
github.com/ebitengine/purego v0.9.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/elastic/gosigar v0.14.3 // indirect
|
||||
github.com/emersion/go-message v0.18.2 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
@@ -428,8 +428,8 @@ require (
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nats.go v1.43.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.11 // indirect
|
||||
github.com/nats-io/nats.go v1.48.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.12 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.5 // indirect
|
||||
@@ -493,11 +493,11 @@ require (
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/otel v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.40.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/term v0.40.0 // indirect
|
||||
|
||||
@@ -6,8 +6,8 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8=
|
||||
atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ=
|
||||
atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs=
|
||||
atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU=
|
||||
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
|
||||
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
|
||||
cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
|
||||
cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
|
||||
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
|
||||
cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
|
||||
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
|
||||
cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc=
|
||||
cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA=
|
||||
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
|
||||
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
|
||||
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
|
||||
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
|
||||
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
|
||||
@@ -338,8 +338,8 @@ cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJP
|
||||
cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk=
|
||||
cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4=
|
||||
cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w=
|
||||
cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM=
|
||||
cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U=
|
||||
cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
|
||||
cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
|
||||
cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA=
|
||||
cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o=
|
||||
cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM=
|
||||
@@ -393,8 +393,8 @@ cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhz
|
||||
cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
|
||||
cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM=
|
||||
cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk=
|
||||
cloud.google.com/go/pubsub/v2 v2.2.1 h1:3brZcshL3fIiD1qOxAE2QW9wxsfjioy014x4yC9XuYI=
|
||||
cloud.google.com/go/pubsub/v2 v2.2.1/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0 h1:DgAN907x+sP0nScYfBzneRiIhWoXcpCD8ZAut8WX9vs=
|
||||
cloud.google.com/go/pubsub/v2 v2.3.0/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
|
||||
cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
|
||||
cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
|
||||
cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
|
||||
@@ -504,8 +504,8 @@ cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV6
|
||||
cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28=
|
||||
cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y=
|
||||
cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA=
|
||||
cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4=
|
||||
cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI=
|
||||
cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
|
||||
cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
|
||||
cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs=
|
||||
cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg=
|
||||
cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos=
|
||||
@@ -687,6 +687,8 @@ github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUS
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op h1:Ucf+QxEKMbPogRO5guBNe5cgd9uZgfoJLOYs8WWhtjM=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
|
||||
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
|
||||
github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4=
|
||||
@@ -714,8 +716,8 @@ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+
|
||||
github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
|
||||
@@ -724,8 +726,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUT
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 h1:2fjfz3/G9BRvIKuNZ655GwzpklC2kEH0cowZQGO7uBg=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4/go.mod h1:Ymws824lvMypLFPwyyUXM52SXuGgxpu0+DISLfKvB+c=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
|
||||
@@ -746,18 +748,18 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZi
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng=
|
||||
github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
@@ -855,8 +857,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
|
||||
github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
|
||||
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
|
||||
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e h1:gt7U1Igw0xbJdyaCM5H2CnlAlPSkzrhsebQB6WQWjLA=
|
||||
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
|
||||
github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
|
||||
github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
|
||||
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
|
||||
@@ -972,8 +974,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
|
||||
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
|
||||
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJtLmY22n99HaZTz+r2Z51xUPi01m3wg=
|
||||
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg=
|
||||
github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
|
||||
@@ -1000,8 +1002,8 @@ github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJ
|
||||
github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
|
||||
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
|
||||
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g=
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98=
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
@@ -1051,8 +1053,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
|
||||
github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
|
||||
github.com/getsentry/sentry-go v0.43.0 h1:XbXLpFicpo8HmBDaInk7dum18G9KSLcjZiyUKS+hLW4=
|
||||
github.com/getsentry/sentry-go v0.43.0/go.mod h1:XDotiNZbgf5U8bPDUAfvcFmOnMQQceESxyKaObSssW0=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
|
||||
github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
|
||||
@@ -1124,8 +1126,8 @@ github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOr
|
||||
github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-redsync/redsync/v4 v4.15.0 h1:KH/XymuxSV7vyKs6z1Cxxj+N+N18JlPxgXeP6x4JY54=
|
||||
github.com/go-redsync/redsync/v4 v4.15.0/go.mod h1:qNp+lLs3vkfZbtA/aM/OjlZHfEr5YTAYhRktFPKHC7s=
|
||||
github.com/go-redsync/redsync/v4 v4.16.0 h1:bNcOzeHH9d3s6pghU9NJFMPrQa41f5Nx3L4YKr3BdEU=
|
||||
github.com/go-redsync/redsync/v4 v4.16.0/go.mod h1:V4gagqgyASWBZuwx4xGzu72aZNb/6Mo05byUa3mVmKQ=
|
||||
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
|
||||
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
@@ -1237,6 +1239,8 @@ github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV
|
||||
github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI=
|
||||
github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
|
||||
github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
|
||||
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
@@ -1514,8 +1518,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
|
||||
github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
|
||||
github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkFS2iI=
|
||||
github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
|
||||
github.com/linkedin/goavro/v2 v2.15.0 h1:pDj1UrjUOO62iXhgBiE7jQkpNIc5/tA5eZsgolMjgVI=
|
||||
github.com/linkedin/goavro/v2 v2.15.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
|
||||
github.com/linxGnu/grocksdb v1.10.7 h1:fCi4qvZWo04VgFwGWmO8HQJgUVounJBy+C2TMVPU/ho=
|
||||
github.com/linxGnu/grocksdb v1.10.7/go.mod h1:OLQKZwiKwaJiAVCsOzWKvwiLwfZ5Vz8Md5TYR7t7pM8=
|
||||
github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=
|
||||
@@ -1563,8 +1567,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
|
||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
|
||||
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
|
||||
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
|
||||
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
|
||||
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
|
||||
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
|
||||
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
@@ -1622,14 +1626,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
|
||||
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
|
||||
github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak=
|
||||
github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
|
||||
github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU=
|
||||
github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0=
|
||||
github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug=
|
||||
github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
|
||||
github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
|
||||
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
|
||||
github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
|
||||
github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
|
||||
github.com/nats-io/nats-server/v2 v2.11.12 h1:jGDXTkcjqQ5fCRstwIxvv1K0RHfftFUoSCT/iIZcqOc=
|
||||
github.com/nats-io/nats-server/v2 v2.11.12/go.mod h1:5MCp/pqm5SEfsvVZ31ll1088ZTwEUdvRX1Hmh/mTTDg=
|
||||
github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U=
|
||||
github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
|
||||
github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
|
||||
github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
@@ -1765,8 +1769,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
|
||||
github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
|
||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/pterm/pterm v0.12.27/go.mod h1:PhQ89w4i95rhgE+xedAoqous6K9X+r6aSOI2eFF7DZI=
|
||||
github.com/pterm/pterm v0.12.29/go.mod h1:WI3qxgvoQFFGKGjGnJR849gU0TsEOvKn5Q8LlY1U7lg=
|
||||
github.com/pterm/pterm v0.12.30/go.mod h1:MOqLIyMOgmTDz9yorcYbcw+HsgoZo3BQfg2wtl3HEFE=
|
||||
@@ -1800,10 +1804,10 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
|
||||
github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.69/go.mod h1:iC4Y8DoN0Uth0Uezg9e2trvNRC7QAgGeuP2OPLb5ccI=
|
||||
github.com/redis/rueidis v1.0.71 h1:pODtnAR5GAB7j4ekhldZ29HKOxe4Hph0GTDGk1ayEQY=
|
||||
github.com/redis/rueidis v1.0.71/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.71 h1:wNZ//kEjMZgBM0KCk7ncOX8KmAgROU2kDdDNpwheG4w=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.71/go.mod h1:esmCLJvaRzZoKlgB82G1bY7Iky5TnO9Rz+NlhbEccFI=
|
||||
github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
|
||||
github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM=
|
||||
github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
|
||||
@@ -1857,8 +1861,8 @@ github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQ
|
||||
github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
|
||||
github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
|
||||
github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
|
||||
github.com/shirou/gopsutil/v4 v4.26.1 h1:TOkEyriIXk2HX9d4isZJtbjXbEjf5qyKPAzbzY0JWSo=
|
||||
github.com/shirou/gopsutil/v4 v4.26.1/go.mod h1:medLI9/UNAb0dOI9Q3/7yWSqKkj00u+1tgY8nvv41pc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
|
||||
github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
|
||||
@@ -2109,8 +2113,8 @@ go.etcd.io/etcd/client/pkg/v3 v3.6.7 h1:vvzgyozz46q+TyeGBuFzVuI53/yd133CHceNb/Ah
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7/go.mod h1:2IVulJ3FZ/czIGl9T4lMF1uxzrhRahLqe+hSgy+Kh7Q=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7 h1:9WqA5RpIBtdMxAy1ukXLAdtg2pAxNqW5NUoO2wQrE6U=
|
||||
go.etcd.io/etcd/client/v3 v3.6.7/go.mod h1:2XfROY56AXnUqGsvl+6k29wrwsSbEh1lAouQB1vHpeE=
|
||||
go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss=
|
||||
go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ=
|
||||
go.mongodb.org/mongo-driver v1.17.9 h1:IexDdCuuNJ3BHrELgBlyaH9p60JXAvdzWR128q+U5tU=
|
||||
go.mongodb.org/mongo-driver v1.17.9/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
@@ -2130,35 +2134,35 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
|
||||
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
|
||||
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0 h1:zG8GlgXCJQd5BU98C0hZnBbElszTmUgCNCfYneaDL0A=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0/go.mod h1:hOfBCz8kv/wuq73Mx2H2QnWokh/kHZxkh6SNF2bdKtw=
|
||||
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
|
||||
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 h1:ZsXq73BERAiNuuFXYqP4MR5hBrjXfMGSO+Cx7qoOZiM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0/go.mod h1:hg1zaDMpyZJuUzjFxFsRYBoccE86tM9Uf4IqNMUxvrY=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI=
|
||||
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
|
||||
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
|
||||
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
|
||||
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
|
||||
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
|
||||
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
|
||||
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
|
||||
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
|
||||
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
|
||||
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
|
||||
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
|
||||
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
|
||||
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
|
||||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
@@ -2182,10 +2186,10 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
gocloud.dev v0.44.0 h1:iVyMAqFl2r6xUy7M4mfqwlN+21UpJoEtgHEcfiLMUXs=
|
||||
gocloud.dev v0.44.0/go.mod h1:ZmjROXGdC/eKZLF1N+RujDlFRx3D+4Av2thREKDMVxY=
|
||||
gocloud.dev/pubsub/natspubsub v0.44.0 h1:1Us76ckkdgtiE1p1rJZ+38b9TQP051bmjAiQlFQzYrM=
|
||||
gocloud.dev/pubsub/natspubsub v0.44.0/go.mod h1:PvVAGIhL14PWGwWIXX/zAK42ixr2/PKP4Q4yMiAUraQ=
|
||||
gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY=
|
||||
gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI=
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0 h1:kfCupVejeynIQcS1GaIvkY3Nj/acLlM5yPWyxZN8wJ8=
|
||||
gocloud.dev/pubsub/natspubsub v0.45.0/go.mod h1:WU5SMDWuF7CCr2U8UpbrEONYYFOmvigCAvTpOOpUvYE=
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.44.0 h1:MpRIO6XJ/JTqrlUWt3CxwDe1LvaiXUVu4sS5cv4f/AM=
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.44.0/go.mod h1:BB9+qT3r6g4M5+4asiXaEeqw4QAOzsWusO5krYaqkdA=
|
||||
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
|
||||
@@ -2376,8 +2380,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
|
||||
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
|
||||
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
|
||||
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
|
||||
golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
|
||||
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
|
||||
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -2505,8 +2509,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 h1:O1cMQHRfwNpDfDJerqRoE2oD+AFlyid87D40L/OkkJo=
|
||||
golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8=
|
||||
@@ -2840,8 +2844,8 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
|
||||
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
|
||||
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
|
||||
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
|
||||
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc=
|
||||
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc=
|
||||
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
|
||||
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 h1:ZdyUkS9po3H7G0tuh955QVyyotWvOD4W0aEapeGeUYk=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846/go.mod h1:Fk4kyraUvqD7i5H6S43sj2W98fbZa75lpZz/eUyhfO0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.13"
|
||||
appVersion: "4.16"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.0.413
|
||||
version: 4.16.0
|
||||
|
||||
@@ -23,8 +23,15 @@
|
||||
#
|
||||
# Adjust storageClass and sizes to match your cluster's available StorageClasses.
|
||||
# On OpenShift you can discover them with: oc get storageclass
|
||||
|
||||
global:
|
||||
enableReplication: true
|
||||
# replication type is XYZ:
|
||||
# X number of replica in other data centers
|
||||
# Y number of replica in other racks in the same data center
|
||||
# Z number of replica in other servers in the same rack
|
||||
replicationPlacement: "000" # no data replica
|
||||
master:
|
||||
replicas: 1
|
||||
data:
|
||||
type: "persistentVolumeClaim"
|
||||
size: "10Gi"
|
||||
@@ -49,6 +56,7 @@ master:
|
||||
type: RuntimeDefault
|
||||
|
||||
volume:
|
||||
replicas: 1
|
||||
dataDirs:
|
||||
- name: data1
|
||||
type: "persistentVolumeClaim"
|
||||
@@ -75,6 +83,7 @@ volume:
|
||||
type: RuntimeDefault
|
||||
|
||||
filer:
|
||||
replicas: 1
|
||||
data:
|
||||
type: "persistentVolumeClaim"
|
||||
size: "25Gi"
|
||||
@@ -100,12 +109,19 @@ filer:
|
||||
|
||||
# S3 gateway (if enabled)
|
||||
s3:
|
||||
enabled: true
|
||||
replicas: 1
|
||||
port: 8333
|
||||
enableAuth: true
|
||||
podSecurityContext:
|
||||
enabled: true
|
||||
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
|
||||
# controller assign them automatically based on the namespace's SCC.
|
||||
runAsNonRoot: true
|
||||
|
||||
logs:
|
||||
type: "emptyDir"
|
||||
|
||||
containerSecurityContext:
|
||||
enabled: true
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -118,8 +118,10 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- if .Values.admin.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.admin.extraEnvironmentVars }}
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.admin "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -128,18 +130,6 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
|
||||
@@ -243,8 +243,7 @@ spec:
|
||||
{{- if $httpsPort }}
|
||||
-s3.port.https={{ $httpsPort }} \
|
||||
{{- end }}
|
||||
-s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/s3/seaweedfs_s3_config \
|
||||
@@ -346,6 +345,9 @@ spec:
|
||||
- name: client-cert
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
readOnly: true
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
{{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.allInOne.extraVolumeMounts . | nindent 12 }}
|
||||
ports:
|
||||
@@ -473,6 +475,9 @@ spec:
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-client-cert
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
{{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.allInOne.extraVolumes . | nindent 8 }}
|
||||
{{- if .Values.allInOne.nodeSelector }}
|
||||
|
||||
@@ -17,6 +17,9 @@ metadata:
|
||||
spec:
|
||||
type: {{ .Values.allInOne.service.type | default "ClusterIP" }}
|
||||
internalTrafficPolicy: {{ .Values.allInOne.service.internalTrafficPolicy | default "Cluster" }}
|
||||
{{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) .Values.allInOne.s3.trafficDistribution }}
|
||||
trafficDistribution: {{ include "seaweedfs.trafficDistribution" (dict "value" .Values.allInOne.s3.trafficDistribution "Capabilities" .Capabilities) }}
|
||||
{{- end }}
|
||||
ports:
|
||||
# Master ports
|
||||
- name: "swfs-master"
|
||||
|
||||
@@ -96,8 +96,10 @@ spec:
|
||||
- name: WEED_GRPC_CA
|
||||
value: /usr/local/share/ca-certificates/client/ca.crt
|
||||
{{- end }}
|
||||
{{- if .Values.cosi.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.cosi.extraEnvironmentVars }}
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.cosi "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -106,18 +108,6 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- mountPath: /var/lib/cosi
|
||||
name: socket
|
||||
|
||||
@@ -114,8 +114,10 @@ spec:
|
||||
optional: true
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- if .Values.filer.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.filer.extraEnvironmentVars }}
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.filer "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -124,18 +126,6 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.secretExtraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
@@ -210,8 +200,7 @@ spec:
|
||||
{{- if .Values.filer.s3.httpsPort }}
|
||||
-s3.port.https={{ .Values.filer.s3.httpsPort }} \
|
||||
{{- end }}
|
||||
-s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "s3.") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.s3.enableAuth }}
|
||||
-s3.config=/etc/sw/seaweedfs_s3_config \
|
||||
@@ -264,6 +253,9 @@ spec:
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
{{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }}
|
||||
ports:
|
||||
@@ -394,6 +386,9 @@ spec:
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-client-cert
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
{{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.filer.extraVolumes . | indent 8 | trim }}
|
||||
{{- if .Values.filer.nodeSelector }}
|
||||
|
||||
@@ -69,9 +69,7 @@ spec:
|
||||
priorityClassName: {{ .Values.master.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
{{- if .Values.global.createClusterRole }}
|
||||
serviceAccountName: {{ .Values.master.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
|
||||
{{- end }}
|
||||
{{- if .Values.master.initContainers }}
|
||||
initContainers:
|
||||
{{ tpl .Values.master.initContainers . | nindent 8 | trim }}
|
||||
@@ -98,8 +96,10 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- if .Values.master.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.master.extraEnvironmentVars }}
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.master "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -108,18 +108,6 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
|
||||
@@ -90,8 +90,10 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- if .Values.s3.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.s3.extraEnvironmentVars }}
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.s3 "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -100,18 +102,6 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
@@ -137,8 +127,7 @@ spec:
|
||||
{{- if .Values.s3.httpsPort }}
|
||||
-port.https={{ .Values.s3.httpsPort }} \
|
||||
{{- end }}
|
||||
-cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{ include "seaweedfs.s3.tlsArgs" (dict "root" . "prefix" "") | nindent 14 }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.domainName }}
|
||||
-domainName={{ .Values.s3.domainName }} \
|
||||
@@ -186,6 +175,7 @@ spec:
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
{{- include "seaweedfs.s3.tlsVolumeMount" . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.s3.extraVolumeMounts . | nindent 12 | trim }}
|
||||
ports:
|
||||
@@ -277,6 +267,7 @@ spec:
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-client-cert
|
||||
{{- include "seaweedfs.s3.tlsVolume" . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{ tpl .Values.s3.extraVolumes . | indent 8 | trim }}
|
||||
{{- if .Values.s3.nodeSelector }}
|
||||
|
||||
@@ -16,8 +16,9 @@ metadata:
|
||||
{{- end }}
|
||||
spec:
|
||||
internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }}
|
||||
{{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) (or .Values.s3.trafficDistribution .Values.filer.s3.trafficDistribution) }}
|
||||
trafficDistribution: {{ include "seaweedfs.trafficDistribution" . }}
|
||||
{{- $td := .Values.s3.trafficDistribution | default .Values.filer.s3.trafficDistribution }}
|
||||
{{- if and (semverCompare ">=1.31-0" .Capabilities.KubeVersion.GitVersion) $td }}
|
||||
trafficDistribution: {{ include "seaweedfs.trafficDistribution" (dict "value" $td "Capabilities" .Capabilities) }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: "swfs-s3"
|
||||
|
||||
@@ -90,8 +90,10 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- if .Values.sftp.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.sftp.extraEnvironmentVars }}
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.sftp "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -100,18 +102,6 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
|
||||
@@ -59,6 +59,18 @@ Inject extra environment vars in the format key:value, if populated
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "seaweedfs.mergeExtraEnvironmentVars" -}}
|
||||
{{- $global := ((.global | default dict).extraEnvironmentVars | default dict) -}}
|
||||
{{- $component := ((.component | default dict).extraEnvironmentVars | default dict) -}}
|
||||
{{- $target := .target -}}
|
||||
{{- range $key, $value := $global }}
|
||||
{{- $_ := set $target $key $value }}
|
||||
{{- end }}
|
||||
{{- range $key, $value := $component }}
|
||||
{{- $_ := set $target $key $value }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Return the proper filer image */}}
|
||||
{{- define "filer.image" -}}
|
||||
{{- if .Values.filer.imageOverride -}}
|
||||
@@ -326,11 +338,41 @@ Create the name of the service account to use
|
||||
{{- .Values.global.serviceAccountName | default "seaweedfs" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Generate a compatible trafficDistribution value due to "PreferClose" fast deprecation in k8s v1.35 */}}
|
||||
{{/* S3 TLS cert/key arguments, using custom secret if s3.tlsSecret is set */}}
|
||||
{{- define "seaweedfs.s3.tlsArgs" -}}
|
||||
{{- $prefix := .prefix -}}
|
||||
{{- $root := .root -}}
|
||||
{{- if $root.Values.s3.tlsSecret -}}
|
||||
-{{ $prefix }}cert.file=/usr/local/share/ca-certificates/s3/tls.crt \
|
||||
-{{ $prefix }}key.file=/usr/local/share/ca-certificates/s3/tls.key \
|
||||
{{- else -}}
|
||||
-{{ $prefix }}cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-{{ $prefix }}key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* S3 custom TLS volume mount */}}
|
||||
{{- define "seaweedfs.s3.tlsVolumeMount" -}}
|
||||
{{- if .Values.s3.tlsSecret }}
|
||||
- name: s3-tls-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/s3/
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* S3 custom TLS volume */}}
|
||||
{{- define "seaweedfs.s3.tlsVolume" -}}
|
||||
{{- if .Values.s3.tlsSecret }}
|
||||
- name: s3-tls-cert
|
||||
secret:
|
||||
secretName: {{ .Values.s3.tlsSecret }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Generate a compatible trafficDistribution value due to "PreferClose" fast deprecation in k8s v1.35.
|
||||
Accepts a dict with "value" (the trafficDistribution string) and "Capabilities". */}}
|
||||
{{- define "seaweedfs.trafficDistribution" -}}
|
||||
{{- if .Values.s3.trafficDistribution -}}
|
||||
{{- and (eq .Values.s3.trafficDistribution "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .Values.s3.trafficDistribution -}}
|
||||
{{- else if .Values.filer.s3.trafficDistribution -}}
|
||||
{{- and (eq .Values.filer.s3.trafficDistribution "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .Values.filer.s3.trafficDistribution -}}
|
||||
{{- if .value -}}
|
||||
{{- and (eq .value "PreferClose") (semverCompare ">=1.35-0" .Capabilities.KubeVersion.GitVersion) | ternary "PreferSameZone" .value -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
@@ -51,7 +51,7 @@ metadata:
|
||||
annotations:
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "-5"
|
||||
"helm.sh/hook-delete-policy": hook-succeeded
|
||||
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
|
||||
spec:
|
||||
template:
|
||||
metadata:
|
||||
@@ -92,6 +92,7 @@ spec:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
- |
|
||||
set -o pipefail
|
||||
wait_for_service() {
|
||||
local url=$1
|
||||
local max_attempts=60 # 5 minutes total (5s * 60)
|
||||
@@ -117,8 +118,7 @@ spec:
|
||||
wait_for_service "http://$WEED_CLUSTER_SW_MASTER{{ .Values.master.readinessProbe.httpGet.path }}"
|
||||
wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}"
|
||||
{{- end }}
|
||||
set -o pipefail
|
||||
{{- range $createBuckets }}
|
||||
{{- range $createBuckets }}
|
||||
{{- $bucketName := .name }}
|
||||
{{- $bucketLock := or .lock .objectLock .withLock }}
|
||||
bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }
|
||||
|
||||
@@ -69,9 +69,7 @@ spec:
|
||||
priorityClassName: {{ $volume.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
{{- if $.Values.global.createClusterRole }}
|
||||
serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
|
||||
{{- end }}
|
||||
{{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
|
||||
{{- if $initContainers_exists }}
|
||||
initContainers:
|
||||
@@ -118,8 +116,10 @@ spec:
|
||||
fieldPath: status.hostIP
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" $ }}"
|
||||
{{- if $volume.extraEnvironmentVars }}
|
||||
{{- range $key, $value := $volume.extraEnvironmentVars }}
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" $.Values.global "component" $volume "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -128,18 +128,6 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if $.Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := $.Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
|
||||
@@ -93,8 +93,10 @@ spec:
|
||||
fieldPath: metadata.namespace
|
||||
- name: SEAWEEDFS_FULLNAME
|
||||
value: "{{ include "seaweedfs.fullname" . }}"
|
||||
{{- if .Values.worker.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.worker.extraEnvironmentVars }}
|
||||
{{- $mergedExtraEnvironmentVars := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.worker "target" $mergedExtraEnvironmentVars) }}
|
||||
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
|
||||
{{- $value := index $mergedExtraEnvironmentVars $key }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
@@ -103,18 +105,6 @@ spec:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
|
||||
@@ -919,6 +919,13 @@ s3:
|
||||
port: 8333
|
||||
# add additional https port
|
||||
httpsPort: 0
|
||||
# Use a custom TLS certificate secret for the S3 HTTPS endpoint.
|
||||
# When set, this Kubernetes Secret (must contain tls.crt and tls.key) is used
|
||||
# instead of the internal self-signed client certificate generated by cert-manager.
|
||||
# This allows using a publicly trusted certificate (e.g., from Let's Encrypt)
|
||||
# so that S3 clients don't need to trust the internal CA.
|
||||
# Requires global.enableSecurity to be true.
|
||||
tlsSecret: null
|
||||
metricsPort: 9327
|
||||
# Iceberg catalog REST port (Apache Iceberg REST Catalog API)
|
||||
# Set to a port number to enable, or 0/null to disable
|
||||
@@ -1453,6 +1460,7 @@ allInOne:
|
||||
# The s3-secret.yaml template only reads from .Values.s3.credentials.
|
||||
# See: s3.credentials.admin.accessKey, s3.credentials.read.accessKey
|
||||
auditLogConfig: null # S3 audit log configuration (null inherits from s3.auditLogConfig)
|
||||
trafficDistribution: null # Service traffic distribution (e.g., "PreferClose"); auto-converts to "PreferSameZone" on k8s >=1.35
|
||||
# You may specify buckets to be created during the install process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
# ttl format: [1-255][m|h|d|w|M|y] (e.g., 7d)
|
||||
|
||||
@@ -100,10 +100,12 @@ message ListEntriesRequest {
|
||||
string startFromFileName = 3;
|
||||
bool inclusiveStartFrom = 4;
|
||||
uint32 limit = 5;
|
||||
int64 snapshot_ts_ns = 6;
|
||||
}
|
||||
|
||||
message ListEntriesResponse {
|
||||
Entry entry = 1;
|
||||
int64 snapshot_ts_ns = 2;
|
||||
}
|
||||
|
||||
message RemoteEntry {
|
||||
@@ -203,6 +205,7 @@ message CreateEntryRequest {
|
||||
|
||||
message CreateEntryResponse {
|
||||
string error = 1;
|
||||
SubscribeMetadataResponse metadata_event = 2;
|
||||
}
|
||||
|
||||
message UpdateEntryRequest {
|
||||
@@ -212,6 +215,7 @@ message UpdateEntryRequest {
|
||||
repeated int32 signatures = 4;
|
||||
}
|
||||
message UpdateEntryResponse {
|
||||
SubscribeMetadataResponse metadata_event = 1;
|
||||
}
|
||||
|
||||
message AppendToEntryRequest {
|
||||
@@ -236,6 +240,7 @@ message DeleteEntryRequest {
|
||||
|
||||
message DeleteEntryResponse {
|
||||
string error = 1;
|
||||
SubscribeMetadataResponse metadata_event = 2;
|
||||
}
|
||||
|
||||
message AtomicRenameEntryRequest {
|
||||
@@ -469,6 +474,7 @@ message CacheRemoteObjectToLocalClusterRequest {
|
||||
}
|
||||
message CacheRemoteObjectToLocalClusterResponse {
|
||||
Entry entry = 1;
|
||||
SubscribeMetadataResponse metadata_event = 2;
|
||||
}
|
||||
|
||||
/////////////////////////
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Multi-stage build for Go Sidecar
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git ca-certificates tzdata
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Multi-stage build for Test Client
|
||||
FROM golang:1.23-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git ca-certificates tzdata
|
||||
|
||||
@@ -153,6 +153,38 @@ func waitForUrl(t *testing.T, url string, retries int) {
|
||||
t.Fatalf("Timeout waiting for %s", url)
|
||||
}
|
||||
|
||||
func fetchJSON(url string, out interface{}) error {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
|
||||
func mapField(obj map[string]interface{}, key string) (interface{}, bool) {
|
||||
if obj == nil {
|
||||
return nil, false
|
||||
}
|
||||
if value, ok := obj[key]; ok {
|
||||
return value, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func mapFieldAny(obj map[string]interface{}, keys ...string) (interface{}, bool) {
|
||||
for _, key := range keys {
|
||||
if value, ok := mapField(obj, key); ok {
|
||||
return value, true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func TestEcEndToEnd(t *testing.T) {
|
||||
defer cleanup()
|
||||
ensureEnvironment(t)
|
||||
@@ -275,6 +307,7 @@ func TestEcEndToEnd(t *testing.T) {
|
||||
startTime := time.Now()
|
||||
ecVerified := false
|
||||
var lastBody []byte
|
||||
debugTick := 0
|
||||
|
||||
for time.Since(startTime) < 300*time.Second {
|
||||
// 3.1 Check Master Topology
|
||||
@@ -300,25 +333,104 @@ func TestEcEndToEnd(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// 3.2 Debug: Check workers and jobs
|
||||
wResp, wErr := http.Get(AdminUrl + "/api/plugin/workers")
|
||||
// 3.2 Debug: Check workers, jobs, and scheduler status
|
||||
debugTick++
|
||||
|
||||
var workers []map[string]interface{}
|
||||
workerCount := 0
|
||||
if wErr == nil {
|
||||
var workers []interface{}
|
||||
json.NewDecoder(wResp.Body).Decode(&workers)
|
||||
wResp.Body.Close()
|
||||
ecDetectorCount := 0
|
||||
ecExecutorCount := 0
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/workers", &workers); err == nil {
|
||||
workerCount = len(workers)
|
||||
for _, worker := range workers {
|
||||
capsValue, ok := mapFieldAny(worker, "capabilities", "Capabilities")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
caps, ok := capsValue.(map[string]interface{})
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if capValue, ok := caps["erasure_coding"].(map[string]interface{}); ok {
|
||||
if capValue["can_detect"] == true {
|
||||
ecDetectorCount++
|
||||
}
|
||||
if capValue["can_execute"] == true {
|
||||
ecExecutorCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tResp, tErr := http.Get(AdminUrl + "/api/plugin/jobs?limit=1000")
|
||||
var tasks []map[string]interface{}
|
||||
taskCount := 0
|
||||
if tErr == nil {
|
||||
var tasks []interface{}
|
||||
json.NewDecoder(tResp.Body).Decode(&tasks)
|
||||
tResp.Body.Close()
|
||||
ecTaskCount := 0
|
||||
ecTaskStates := map[string]int{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/jobs?limit=1000", &tasks); err == nil {
|
||||
taskCount = len(tasks)
|
||||
for _, task := range tasks {
|
||||
jobType, _ := task["job_type"].(string)
|
||||
state, _ := task["state"].(string)
|
||||
if jobType == "erasure_coding" {
|
||||
ecTaskCount++
|
||||
ecTaskStates[state]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("Waiting for EC... (Workers: %d det=%d exec=%d, Tasks: %d ec=%d, EC States: %+v)",
|
||||
workerCount, ecDetectorCount, ecExecutorCount, taskCount, ecTaskCount, ecTaskStates)
|
||||
|
||||
if debugTick%3 == 0 {
|
||||
var pluginStatus map[string]interface{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/status", &pluginStatus); err == nil {
|
||||
t.Logf("Plugin status: enabled=%v worker_count=%v worker_grpc_port=%v configured=%v",
|
||||
pluginStatus["enabled"], pluginStatus["worker_count"], pluginStatus["worker_grpc_port"], pluginStatus["configured"])
|
||||
}
|
||||
|
||||
var schedulerStatus map[string]interface{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-status", &schedulerStatus); err == nil {
|
||||
if schedValue, ok := schedulerStatus["scheduler"].(map[string]interface{}); ok {
|
||||
t.Logf("Scheduler status: current_job_type=%v phase=%v last_iteration_had_jobs=%v idle_sleep_seconds=%v last_iteration_done_at=%v next_detection_at=%v",
|
||||
schedValue["current_job_type"], schedValue["current_phase"],
|
||||
schedValue["last_iteration_had_jobs"], schedValue["idle_sleep_seconds"], schedValue["last_iteration_done_at"], schedValue["next_detection_at"])
|
||||
} else {
|
||||
t.Logf("Scheduler status: %v", schedulerStatus)
|
||||
}
|
||||
}
|
||||
|
||||
var schedulerStates []map[string]interface{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-states", &schedulerStates); err == nil {
|
||||
for _, state := range schedulerStates {
|
||||
if state["job_type"] == "erasure_coding" {
|
||||
t.Logf("EC scheduler state: enabled=%v detection_in_flight=%v detector_available=%v executor_workers=%v next_detection_at=%v last_run_status=%v last_run_started_at=%v last_run_completed_at=%v",
|
||||
state["enabled"], state["detection_in_flight"], state["detector_available"],
|
||||
state["executor_worker_count"], state["next_detection_at"], state["last_run_status"],
|
||||
state["last_run_started_at"], state["last_run_completed_at"])
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var jobTypes []map[string]interface{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/job-types", &jobTypes); err == nil {
|
||||
var names []string
|
||||
for _, jobType := range jobTypes {
|
||||
if name, ok := jobType["job_type"].(string); ok && name != "" {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
t.Logf("Plugin job types: %v", names)
|
||||
}
|
||||
|
||||
var activities []map[string]interface{}
|
||||
if err := fetchJSON(AdminUrl+"/api/plugin/activities?job_type=erasure_coding&limit=5", &activities); err == nil {
|
||||
for i := len(activities) - 1; i >= 0; i-- {
|
||||
act := activities[i]
|
||||
t.Logf("EC activity: stage=%v message=%v occurred_at=%v", act["stage"], act["message"], act["occurred_at"])
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Logf("Waiting for EC... (Workers: %d, Active Tasks: %d)", workerCount, taskCount)
|
||||
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Simplified single-stage build for SeaweedFS with FoundationDB support
|
||||
# Force x86_64 platform to use AMD64 FoundationDB packages
|
||||
FROM --platform=linux/amd64 golang:1.24-bookworm
|
||||
FROM --platform=linux/amd64 golang:1.25-bookworm
|
||||
|
||||
ARG FOUNDATIONDB_VERSION=7.4.5
|
||||
ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Multi-stage Dockerfile to build SeaweedFS with FoundationDB support for ARM64
|
||||
FROM --platform=linux/arm64 golang:1.24-bookworm AS builder
|
||||
FROM --platform=linux/arm64 golang:1.25-bookworm AS builder
|
||||
|
||||
ARG FOUNDATIONDB_VERSION=7.4.5
|
||||
ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Test environment with Go and FoundationDB support
|
||||
FROM golang:1.24-bookworm
|
||||
FROM golang:1.25-bookworm
|
||||
|
||||
# Install system dependencies
|
||||
RUN apt-get update && apt-get install -y \
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
<scala.binary.version>2.12</scala.binary.version>
|
||||
<junit.version>4.13.2</junit.version>
|
||||
<seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
|
||||
<jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
|
||||
<jackson.version>2.18.6</jackson.version> <!-- Upgraded from 2.15.3 -->
|
||||
<netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
|
||||
<parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
|
||||
<parquet.format.version>2.12.0</parquet.format.version>
|
||||
@@ -123,7 +123,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.zookeeper</groupId>
|
||||
<artifactId>zookeeper</artifactId>
|
||||
<version>3.9.4</version>
|
||||
<version>3.9.5</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Apache Commons - Fix CVEs -->
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Dockerfile for Kafka Gateway Integration Testing
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git make gcc musl-dev sqlite-dev
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Dockerfile for building SeaweedFS components from the current workspace
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
RUN apk add --no-cache git make gcc musl-dev sqlite-dev
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Dockerfile for Kafka Integration Test Setup
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git make gcc musl-dev
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# Multi-stage build for cross-platform support
|
||||
|
||||
# Stage 1: Builder
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
||||
@@ -27,19 +27,21 @@ type VolumeServer struct {
|
||||
address string
|
||||
baseDir string
|
||||
|
||||
mu sync.Mutex
|
||||
receivedFiles map[string]uint64
|
||||
mountRequests []*volume_server_pb.VolumeEcShardsMountRequest
|
||||
deleteRequests []*volume_server_pb.VolumeDeleteRequest
|
||||
markReadonlyCalls int
|
||||
vacuumGarbageRatio float64
|
||||
vacuumCheckCalls int
|
||||
vacuumCompactCalls int
|
||||
vacuumCommitCalls int
|
||||
vacuumCleanupCalls int
|
||||
volumeCopyCalls int
|
||||
volumeMountCalls int
|
||||
tailReceiverCalls int
|
||||
mu sync.Mutex
|
||||
receivedFiles map[string]uint64
|
||||
mountRequests []*volume_server_pb.VolumeEcShardsMountRequest
|
||||
deleteRequests []*volume_server_pb.VolumeDeleteRequest
|
||||
markReadonlyCalls int
|
||||
markWritableCalls int
|
||||
readFileStatusCalls int
|
||||
vacuumGarbageRatio float64
|
||||
vacuumCheckCalls int
|
||||
vacuumCompactCalls int
|
||||
vacuumCommitCalls int
|
||||
vacuumCleanupCalls int
|
||||
volumeCopyCalls int
|
||||
volumeMountCalls int
|
||||
tailReceiverCalls int
|
||||
}
|
||||
|
||||
// NewVolumeServer starts a test volume server using the provided base directory.
|
||||
@@ -151,6 +153,20 @@ func (v *VolumeServer) MarkReadonlyCount() int {
|
||||
return v.markReadonlyCalls
|
||||
}
|
||||
|
||||
// MarkWritableCount returns the number of writable calls.
|
||||
func (v *VolumeServer) MarkWritableCount() int {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
return v.markWritableCalls
|
||||
}
|
||||
|
||||
// ReadFileStatusCount returns the number of ReadVolumeFileStatus calls.
|
||||
func (v *VolumeServer) ReadFileStatusCount() int {
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
return v.readFileStatusCalls
|
||||
}
|
||||
|
||||
// Shutdown stops the volume server.
|
||||
func (v *VolumeServer) Shutdown() {
|
||||
if v.server != nil {
|
||||
@@ -280,6 +296,25 @@ func (v *VolumeServer) VolumeMarkReadonly(ctx context.Context, req *volume_serve
|
||||
return &volume_server_pb.VolumeMarkReadonlyResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VolumeMarkWritable(ctx context.Context, req *volume_server_pb.VolumeMarkWritableRequest) (*volume_server_pb.VolumeMarkWritableResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.markWritableCalls++
|
||||
v.mu.Unlock()
|
||||
return &volume_server_pb.VolumeMarkWritableResponse{}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) ReadVolumeFileStatus(ctx context.Context, req *volume_server_pb.ReadVolumeFileStatusRequest) (*volume_server_pb.ReadVolumeFileStatusResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.readFileStatusCalls++
|
||||
v.mu.Unlock()
|
||||
return &volume_server_pb.ReadVolumeFileStatusResponse{
|
||||
VolumeId: req.VolumeId,
|
||||
DatFileSize: 1024,
|
||||
IdxFileSize: 16,
|
||||
FileCount: 1,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (v *VolumeServer) VacuumVolumeCheck(ctx context.Context, req *volume_server_pb.VacuumVolumeCheckRequest) (*volume_server_pb.VacuumVolumeCheckResponse, error) {
|
||||
v.mu.Lock()
|
||||
v.vacuumCheckCalls++
|
||||
|
||||
@@ -37,7 +37,9 @@ func TestVolumeBalanceDetectionIntegration(t *testing.T) {
|
||||
MasterGrpcAddresses: []string{master.Address()},
|
||||
}, 10)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, proposals, 1)
|
||||
// With default batch_size=20 and 10 overloaded volumes vs 1 underloaded,
|
||||
// all moves are grouped into a single batch proposal.
|
||||
require.Len(t, proposals, 1, "expected exactly one batch proposal")
|
||||
|
||||
proposal := proposals[0]
|
||||
require.Equal(t, "volume_balance", proposal.JobType)
|
||||
@@ -46,8 +48,15 @@ func TestVolumeBalanceDetectionIntegration(t *testing.T) {
|
||||
|
||||
params := &worker_pb.TaskParams{}
|
||||
require.NoError(t, proto.Unmarshal(paramsValue.GetBytesValue(), params))
|
||||
require.NotEmpty(t, params.Sources)
|
||||
require.NotEmpty(t, params.Targets)
|
||||
|
||||
bp := params.GetBalanceParams()
|
||||
require.NotNil(t, bp, "expected BalanceParams in batch proposal")
|
||||
require.Greater(t, len(bp.Moves), 1, "batch proposal should contain multiple moves")
|
||||
for _, move := range bp.Moves {
|
||||
require.NotZero(t, move.VolumeId)
|
||||
require.NotEmpty(t, move.SourceNode)
|
||||
require.NotEmpty(t, move.TargetNode)
|
||||
}
|
||||
}
|
||||
|
||||
func buildBalanceVolumeListResponse(t *testing.T) *master_pb.VolumeListResponse {
|
||||
|
||||
@@ -8,10 +8,12 @@ import (
|
||||
|
||||
pluginworkers "github.com/seaweedfs/seaweedfs/test/plugin_workers"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
||||
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func TestVolumeBalanceExecutionIntegration(t *testing.T) {
|
||||
@@ -60,8 +62,92 @@ func TestVolumeBalanceExecutionIntegration(t *testing.T) {
|
||||
require.GreaterOrEqual(t, source.MarkReadonlyCount(), 1)
|
||||
require.GreaterOrEqual(t, len(source.DeleteRequests()), 1)
|
||||
|
||||
copyCalls, mountCalls, tailCalls := target.BalanceStats()
|
||||
copyCalls, _, tailCalls := target.BalanceStats()
|
||||
require.GreaterOrEqual(t, copyCalls, 1)
|
||||
require.GreaterOrEqual(t, mountCalls, 1)
|
||||
require.GreaterOrEqual(t, tailCalls, 1)
|
||||
}
|
||||
|
||||
func TestVolumeBalanceBatchExecutionIntegration(t *testing.T) {
|
||||
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
|
||||
handler := pluginworker.NewVolumeBalanceHandler(dialOption)
|
||||
harness := pluginworkers.NewHarness(t, pluginworkers.HarnessConfig{
|
||||
WorkerOptions: pluginworker.WorkerOptions{
|
||||
GrpcDialOption: dialOption,
|
||||
},
|
||||
Handlers: []pluginworker.JobHandler{handler},
|
||||
})
|
||||
harness.WaitForJobType("volume_balance")
|
||||
|
||||
// Create one source and one target fake volume server.
|
||||
source := pluginworkers.NewVolumeServer(t, "")
|
||||
target := pluginworkers.NewVolumeServer(t, "")
|
||||
|
||||
// Build a batch job with 3 volume moves from source → target.
|
||||
volumeIDs := []uint32{401, 402, 403}
|
||||
moves := make([]*worker_pb.BalanceMoveSpec, len(volumeIDs))
|
||||
for i, vid := range volumeIDs {
|
||||
moves[i] = &worker_pb.BalanceMoveSpec{
|
||||
VolumeId: vid,
|
||||
SourceNode: source.Address(),
|
||||
TargetNode: target.Address(),
|
||||
Collection: "batch-test",
|
||||
}
|
||||
}
|
||||
|
||||
params := &worker_pb.TaskParams{
|
||||
TaskId: "batch-balance-test",
|
||||
TaskParams: &worker_pb.TaskParams_BalanceParams{
|
||||
BalanceParams: &worker_pb.BalanceTaskParams{
|
||||
MaxConcurrentMoves: 2,
|
||||
Moves: moves,
|
||||
},
|
||||
},
|
||||
}
|
||||
paramBytes, err := proto.Marshal(params)
|
||||
require.NoError(t, err)
|
||||
|
||||
job := &plugin_pb.JobSpec{
|
||||
JobId: "batch-balance-test",
|
||||
JobType: "volume_balance",
|
||||
Parameters: map[string]*plugin_pb.ConfigValue{
|
||||
"task_params_pb": {
|
||||
Kind: &plugin_pb.ConfigValue_BytesValue{BytesValue: paramBytes},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result, err := harness.Plugin().ExecuteJob(ctx, job, nil, 1)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.True(t, result.Success, "batch balance job should succeed; result: %+v", result)
|
||||
|
||||
// Each of the 3 moves should have marked the source readonly and deleted.
|
||||
require.Equal(t, len(volumeIDs), source.MarkReadonlyCount(),
|
||||
"each move should mark source volume readonly")
|
||||
require.Equal(t, len(volumeIDs), len(source.DeleteRequests()),
|
||||
"each move should delete the source volume")
|
||||
|
||||
// Verify delete requests reference the expected volume IDs.
|
||||
deletedVols := make(map[uint32]bool)
|
||||
for _, req := range source.DeleteRequests() {
|
||||
deletedVols[req.VolumeId] = true
|
||||
}
|
||||
for _, vid := range volumeIDs {
|
||||
require.True(t, deletedVols[vid], "volume %d should have been deleted from source", vid)
|
||||
}
|
||||
|
||||
// Pre-delete verification should have called ReadVolumeFileStatus on both
|
||||
// source and target for each volume.
|
||||
require.Equal(t, len(volumeIDs), source.ReadFileStatusCount(),
|
||||
"each move should read source volume status before delete")
|
||||
require.Equal(t, len(volumeIDs), target.ReadFileStatusCount(),
|
||||
"each move should read target volume status before delete")
|
||||
|
||||
// Target should have received copy and tail calls for all 3 volumes.
|
||||
copyCalls, _, tailCalls := target.BalanceStats()
|
||||
require.Equal(t, len(volumeIDs), copyCalls, "target should receive one copy per volume")
|
||||
require.Equal(t, len(volumeIDs), tailCalls, "target should receive one tail per volume")
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
# Set working directory
|
||||
WORKDIR /app
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.24-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
# Install git and other build dependencies
|
||||
RUN apk add --no-cache git make
|
||||
|
||||
@@ -140,7 +140,7 @@ func TestCORSConfigurationManagement(t *testing.T) {
|
||||
Bucket: aws.String(bucketName),
|
||||
CORSConfiguration: corsConfig,
|
||||
})
|
||||
assert.NoError(t, err, "Should be able to put CORS configuration")
|
||||
require.NoError(t, err, "Should be able to put CORS configuration")
|
||||
|
||||
// Wait for metadata subscription to update cache
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
@@ -149,9 +149,9 @@ func TestCORSConfigurationManagement(t *testing.T) {
|
||||
getResp, err := client.GetBucketCors(context.TODO(), &s3.GetBucketCorsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
assert.NoError(t, err, "Should be able to get CORS configuration")
|
||||
assert.NotNil(t, getResp.CORSRules, "CORS configuration should not be nil")
|
||||
assert.Len(t, getResp.CORSRules, 1, "Should have one CORS rule")
|
||||
require.NoError(t, err, "Should be able to get CORS configuration")
|
||||
require.NotNil(t, getResp.CORSRules, "CORS configuration should not be nil")
|
||||
require.Len(t, getResp.CORSRules, 1, "Should have one CORS rule")
|
||||
|
||||
rule := getResp.CORSRules[0]
|
||||
assert.Equal(t, []string{"*"}, rule.AllowedHeaders, "Allowed headers should match")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Multi-stage build for SeaweedFS S3 with IAM
|
||||
FROM golang:1.23-alpine AS builder
|
||||
FROM golang:1.25-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git make curl wget
|
||||
|
||||
@@ -185,6 +185,9 @@ test-context: ## Test only contextual policy enforcement
|
||||
test-presigned: ## Test only presigned URL integration
|
||||
go test -v -run TestS3IAMPresignedURLIntegration ./...
|
||||
|
||||
test-group: ## Run IAM group management tests
|
||||
go test -v -run "TestIAMGroup" ./...
|
||||
|
||||
test-sts: ## Run all STS tests
|
||||
go test -v -run "TestSTS" ./...
|
||||
|
||||
@@ -263,7 +266,7 @@ docker-build: ## Build custom SeaweedFS image for Docker tests
|
||||
|
||||
# All PHONY targets
|
||||
.PHONY: test test-quick run-tests setup start-services stop-services wait-for-services clean logs status debug
|
||||
.PHONY: test-auth test-policy test-expiration test-multipart test-bucket-policy test-context test-presigned test-sts test-sts-assume-role test-sts-ldap
|
||||
.PHONY: test-auth test-policy test-expiration test-multipart test-bucket-policy test-context test-presigned test-group test-sts test-sts-assume-role test-sts-ldap
|
||||
.PHONY: benchmark ci watch install-deps docker-test docker-up docker-down docker-logs docker-build
|
||||
.PHONY: test-distributed test-performance test-stress test-versioning-stress test-keycloak-full test-all-previously-skipped setup-all-tests help-advanced
|
||||
|
||||
|
||||
@@ -0,0 +1,792 @@
|
||||
package iam
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/iam"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestIAMGroupLifecycle tests the full lifecycle of group management:
|
||||
// CreateGroup, GetGroup, ListGroups, DeleteGroup
|
||||
func TestIAMGroupLifecycle(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-group-lifecycle"
|
||||
|
||||
t.Run("create_group", func(t *testing.T) {
|
||||
resp, err := iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, groupName, *resp.Group.GroupName)
|
||||
})
|
||||
|
||||
t.Run("get_group", func(t *testing.T) {
|
||||
resp, err := iamClient.GetGroup(&iam.GetGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, groupName, *resp.Group.GroupName)
|
||||
})
|
||||
|
||||
t.Run("list_groups_contains_created", func(t *testing.T) {
|
||||
resp, err := iamClient.ListGroups(&iam.ListGroupsInput{})
|
||||
require.NoError(t, err)
|
||||
found := false
|
||||
for _, g := range resp.Groups {
|
||||
if *g.GroupName == groupName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Created group should appear in ListGroups")
|
||||
})
|
||||
|
||||
t.Run("create_duplicate_group_fails", func(t *testing.T) {
|
||||
_, err := iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
assert.Error(t, err, "Creating a duplicate group should fail")
|
||||
})
|
||||
|
||||
t.Run("delete_group", func(t *testing.T) {
|
||||
_, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify it's gone
|
||||
resp, err := iamClient.ListGroups(&iam.ListGroupsInput{})
|
||||
require.NoError(t, err)
|
||||
for _, g := range resp.Groups {
|
||||
assert.NotEqual(t, groupName, *g.GroupName,
|
||||
"Deleted group should not appear in ListGroups")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("delete_nonexistent_group_fails", func(t *testing.T) {
|
||||
_, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{
|
||||
GroupName: aws.String("nonexistent-group-xyz"),
|
||||
})
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupMembership tests adding and removing users from groups
|
||||
func TestIAMGroupMembership(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-group-members"
|
||||
userName := "test-user-for-group"
|
||||
|
||||
// Setup: create group and user
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
|
||||
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
|
||||
|
||||
t.Run("add_user_to_group", func(t *testing.T) {
|
||||
_, err := iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("get_group_shows_member", func(t *testing.T) {
|
||||
resp, err := iamClient.GetGroup(&iam.GetGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
found := false
|
||||
for _, u := range resp.Users {
|
||||
if *u.UserName == userName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Added user should appear in GetGroup members")
|
||||
})
|
||||
|
||||
t.Run("list_groups_for_user", func(t *testing.T) {
|
||||
resp, err := iamClient.ListGroupsForUser(&iam.ListGroupsForUserInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
found := false
|
||||
for _, g := range resp.Groups {
|
||||
if *g.GroupName == groupName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Group should appear in ListGroupsForUser")
|
||||
})
|
||||
|
||||
t.Run("add_duplicate_member_is_idempotent", func(t *testing.T) {
|
||||
_, err := iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
// Should succeed (idempotent) or return a benign error
|
||||
// AWS IAM allows duplicate add without error
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("remove_user_from_group", func(t *testing.T) {
|
||||
_, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify removal
|
||||
resp, err := iamClient.GetGroup(&iam.GetGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
for _, u := range resp.Users {
|
||||
assert.NotEqual(t, userName, *u.UserName,
|
||||
"Removed user should not appear in group members")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupPolicyAttachment tests attaching and detaching policies from groups
|
||||
func TestIAMGroupPolicyAttachment(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-group-policies"
|
||||
policyName := "test-group-attach-policy"
|
||||
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:ListBucket","Resource":"*"}]}`
|
||||
|
||||
// Setup: create group and policy
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
policyArn := createPolicyResp.Policy.Arn
|
||||
|
||||
// Cleanup in correct order: detach policy, delete group, delete policy
|
||||
t.Cleanup(func() {
|
||||
if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: policyArn,
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to detach group policy: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete group: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn}); err != nil {
|
||||
t.Logf("cleanup: failed to delete policy: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("attach_group_policy", func(t *testing.T) {
|
||||
_, err := iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: policyArn,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("list_attached_group_policies", func(t *testing.T) {
|
||||
resp, err := iamClient.ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
found := false
|
||||
for _, p := range resp.AttachedPolicies {
|
||||
if *p.PolicyName == policyName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Attached policy should appear in ListAttachedGroupPolicies")
|
||||
})
|
||||
|
||||
t.Run("detach_group_policy", func(t *testing.T) {
|
||||
_, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: policyArn,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify detachment
|
||||
resp, err := iamClient.ListAttachedGroupPolicies(&iam.ListAttachedGroupPoliciesInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
for _, p := range resp.AttachedPolicies {
|
||||
assert.NotEqual(t, policyName, *p.PolicyName,
|
||||
"Detached policy should not appear in ListAttachedGroupPolicies")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupPolicyEnforcement tests that group policies are enforced during S3 operations.
|
||||
// Creates a user with no direct policies, adds them to a group with S3 access,
|
||||
// and verifies they can access S3 through the group policy.
|
||||
func TestIAMGroupPolicyEnforcement(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-enforcement-group"
|
||||
userName := "test-enforcement-user"
|
||||
policyName := "test-enforcement-policy"
|
||||
bucketName := "test-group-enforce-bucket"
|
||||
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"]}]}`
|
||||
|
||||
// Create user
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create access key for the user
|
||||
keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
accessKeyId := *keyResp.AccessKey.AccessKeyId
|
||||
secretKey := *keyResp.AccessKey.SecretAccessKey
|
||||
|
||||
// Create an S3 client with the user's credentials
|
||||
userS3Client := createS3Client(t, accessKeyId, secretKey)
|
||||
|
||||
// Create group
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create policy
|
||||
createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
policyArn := createPolicyResp.Policy.Arn
|
||||
|
||||
// Cleanup in correct order: remove user from group, detach policy,
|
||||
// delete access key, delete user, delete group, delete policy
|
||||
t.Cleanup(func() {
|
||||
if _, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to remove user from group: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: policyArn,
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to detach group policy: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
|
||||
UserName: aws.String(userName),
|
||||
AccessKeyId: keyResp.AccessKey.AccessKeyId,
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to delete access key: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete user: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete group: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: policyArn}); err != nil {
|
||||
t.Logf("cleanup: failed to delete policy: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Register bucket cleanup on parent test with admin credentials
|
||||
// (userS3Client may lack permissions by cleanup time)
|
||||
adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
if _, err := adminS3.DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("test-key"),
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to delete object: %v", err)
|
||||
}
|
||||
if _, err := adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete bucket: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("user_without_group_denied", func(t *testing.T) {
|
||||
// User has no policies and is not in any group — should be denied
|
||||
_, err := userS3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.Error(t, err, "User without any policies should be denied")
|
||||
awsErr, ok := err.(awserr.Error)
|
||||
require.True(t, ok, "Expected awserr.Error")
|
||||
assert.Equal(t, "AccessDenied", awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("user_with_group_policy_allowed", func(t *testing.T) {
|
||||
// Attach policy to group
|
||||
_, err := iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: policyArn,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Add user to group
|
||||
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait for policy propagation, then create bucket
|
||||
require.Eventually(t, func() bool {
|
||||
_, err = userS3Client.CreateBucket(&s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return err == nil
|
||||
}, 10*time.Second, 500*time.Millisecond, "User with group policy should be allowed")
|
||||
|
||||
// Should also be able to put/get objects
|
||||
_, err = userS3Client.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("test-key"),
|
||||
Body: aws.ReadSeekCloser(strings.NewReader("test-data")),
|
||||
})
|
||||
require.NoError(t, err, "User should be able to put objects through group policy")
|
||||
})
|
||||
|
||||
t.Run("user_removed_from_group_denied", func(t *testing.T) {
|
||||
// Remove user from group
|
||||
_, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait for policy propagation — user should now be denied
|
||||
var lastErr error
|
||||
require.Eventually(t, func() bool {
|
||||
_, lastErr = userS3Client.ListObjects(&s3.ListObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return lastErr != nil
|
||||
}, 10*time.Second, 500*time.Millisecond, "User removed from group should be denied")
|
||||
awsErr, ok := lastErr.(awserr.Error)
|
||||
require.True(t, ok, "Expected awserr.Error")
|
||||
assert.Equal(t, "AccessDenied", awsErr.Code())
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupDisabledPolicyEnforcement tests that disabled groups do not contribute policies.
|
||||
// Uses the raw IAM API (callIAMAPI) since the AWS SDK doesn't support custom group status.
|
||||
func TestIAMGroupDisabledPolicyEnforcement(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
if !isSeaweedFSRunning(t) {
|
||||
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
|
||||
}
|
||||
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-disabled-group"
|
||||
userName := "test-disabled-grp-user"
|
||||
policyName := "test-disabled-grp-policy"
|
||||
bucketName := "test-disabled-grp-bucket"
|
||||
policyDoc := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::` + bucketName + `","arn:aws:s3:::` + bucketName + `/*"]}]}`
|
||||
|
||||
// Create user, group, policy
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
keyResp, err := iamClient.CreateAccessKey(&iam.CreateAccessKeyInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(groupName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
createPolicyResp, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName), PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Cleanup in correct order: remove user from group, detach policy,
|
||||
// delete access key, delete user, delete group, delete policy
|
||||
t.Cleanup(func() {
|
||||
if _, err := iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(groupName), UserName: aws.String(userName),
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to remove user from group: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DetachGroupPolicy(&iam.DetachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName),
|
||||
PolicyArn: aws.String("arn:aws:iam:::policy/" + policyName),
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to detach group policy: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteAccessKey(&iam.DeleteAccessKeyInput{
|
||||
UserName: aws.String(userName), AccessKeyId: keyResp.AccessKey.AccessKeyId,
|
||||
}); err != nil {
|
||||
t.Logf("cleanup: failed to delete access key: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete user: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)}); err != nil {
|
||||
t.Logf("cleanup: failed to delete group: %v", err)
|
||||
}
|
||||
if _, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: createPolicyResp.Policy.Arn}); err != nil {
|
||||
t.Logf("cleanup: failed to delete policy: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// Setup: attach policy, add user, create bucket with admin
|
||||
_, err = iamClient.AttachGroupPolicy(&iam.AttachGroupPolicyInput{
|
||||
GroupName: aws.String(groupName), PolicyArn: createPolicyResp.Policy.Arn,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(groupName), UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
userS3Client := createS3Client(t, *keyResp.AccessKey.AccessKeyId, *keyResp.AccessKey.SecretAccessKey)
|
||||
|
||||
// Create bucket using admin first so we can test listing
|
||||
adminS3, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
_, err = adminS3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucketName)})
|
||||
require.NoError(t, err)
|
||||
defer adminS3.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucketName)})
|
||||
|
||||
t.Run("enabled_group_allows_access", func(t *testing.T) {
|
||||
require.Eventually(t, func() bool {
|
||||
_, err := userS3Client.ListObjects(&s3.ListObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return err == nil
|
||||
}, 10*time.Second, 500*time.Millisecond, "User in enabled group should have access")
|
||||
})
|
||||
|
||||
t.Run("disabled_group_denies_access", func(t *testing.T) {
|
||||
// Disable group via raw IAM API (no SDK support for this extension)
|
||||
resp, err := callIAMAPIAuthenticated(t, framework, "UpdateGroup", url.Values{
|
||||
"GroupName": {groupName},
|
||||
"Disabled": {"true"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "UpdateGroup (disable) should return 200")
|
||||
|
||||
// Wait for propagation — user should be denied
|
||||
var lastErr error
|
||||
require.Eventually(t, func() bool {
|
||||
_, lastErr = userS3Client.ListObjects(&s3.ListObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return lastErr != nil
|
||||
}, 10*time.Second, 500*time.Millisecond, "User in disabled group should be denied access")
|
||||
awsErr, ok := lastErr.(awserr.Error)
|
||||
require.True(t, ok, "Expected awserr.Error")
|
||||
assert.Equal(t, "AccessDenied", awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("re_enabled_group_restores_access", func(t *testing.T) {
|
||||
// Re-enable the group
|
||||
resp, err := callIAMAPIAuthenticated(t, framework, "UpdateGroup", url.Values{
|
||||
"GroupName": {groupName},
|
||||
"Disabled": {"false"},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, http.StatusOK, resp.StatusCode, "UpdateGroup (re-enable) should return 200")
|
||||
|
||||
// Wait for propagation — user should have access again
|
||||
require.Eventually(t, func() bool {
|
||||
_, err = userS3Client.ListObjects(&s3.ListObjectsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return err == nil
|
||||
}, 10*time.Second, 500*time.Millisecond, "User in re-enabled group should have access again")
|
||||
})
|
||||
}
|
||||
|
||||
// TestIAMGroupUserDeletionSideEffect tests that deleting a user removes them from all groups.
|
||||
func TestIAMGroupUserDeletionSideEffect(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
groupName := "test-deletion-group"
|
||||
userName := "test-deletion-user"
|
||||
|
||||
// Create group and user
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(groupName)})
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(groupName)})
|
||||
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
// Best-effort: user may already be deleted by the test
|
||||
iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
|
||||
})
|
||||
|
||||
// Add user to group
|
||||
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(groupName),
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify user is in group
|
||||
getResp, err := iamClient.GetGroup(&iam.GetGroupInput{GroupName: aws.String(groupName)})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, getResp.Users, 1, "Group should have 1 member before deletion")
|
||||
|
||||
// Delete the user
|
||||
_, err = iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify user was removed from the group
|
||||
getResp, err = iamClient.GetGroup(&iam.GetGroupInput{GroupName: aws.String(groupName)})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, getResp.Users, "Group should have no members after user deletion")
|
||||
}
|
||||
|
||||
// TestIAMGroupMultipleGroups tests that a user can belong to multiple groups
|
||||
// and inherits policies from all of them.
|
||||
func TestIAMGroupMultipleGroups(t *testing.T) {
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
iamClient, err := framework.CreateIAMClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
group1 := "test-multi-group-1"
|
||||
group2 := "test-multi-group-2"
|
||||
userName := "test-multi-group-user"
|
||||
|
||||
// Create two groups
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(group1)})
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(group1)})
|
||||
|
||||
_, err = iamClient.CreateGroup(&iam.CreateGroupInput{GroupName: aws.String(group2)})
|
||||
require.NoError(t, err)
|
||||
defer iamClient.DeleteGroup(&iam.DeleteGroupInput{GroupName: aws.String(group2)})
|
||||
|
||||
// Create user
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(group1), UserName: aws.String(userName),
|
||||
})
|
||||
iamClient.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
|
||||
GroupName: aws.String(group2), UserName: aws.String(userName),
|
||||
})
|
||||
iamClient.DeleteUser(&iam.DeleteUserInput{UserName: aws.String(userName)})
|
||||
}()
|
||||
|
||||
// Add user to both groups
|
||||
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(group1), UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.AddUserToGroup(&iam.AddUserToGroupInput{
|
||||
GroupName: aws.String(group2), UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify user appears in both groups
|
||||
resp, err := iamClient.ListGroupsForUser(&iam.ListGroupsForUserInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
groupNames := make(map[string]bool)
|
||||
for _, g := range resp.Groups {
|
||||
groupNames[*g.GroupName] = true
|
||||
}
|
||||
assert.True(t, groupNames[group1], "User should be in group 1")
|
||||
assert.True(t, groupNames[group2], "User should be in group 2")
|
||||
}
|
||||
|
||||
// --- Response types for raw IAM API calls ---
|
||||
|
||||
type CreateGroupResponse struct {
|
||||
XMLName xml.Name `xml:"CreateGroupResponse"`
|
||||
CreateGroupResult struct {
|
||||
Group struct {
|
||||
GroupName string `xml:"GroupName"`
|
||||
} `xml:"Group"`
|
||||
} `xml:"CreateGroupResult"`
|
||||
}
|
||||
|
||||
type ListGroupsResponse struct {
|
||||
XMLName xml.Name `xml:"ListGroupsResponse"`
|
||||
ListGroupsResult struct {
|
||||
Groups []struct {
|
||||
GroupName string `xml:"GroupName"`
|
||||
} `xml:"Groups>member"`
|
||||
} `xml:"ListGroupsResult"`
|
||||
}
|
||||
|
||||
// callIAMAPIAuthenticated sends an authenticated raw IAM API request using the
|
||||
// framework's JWT token. This is needed for custom extensions not in the AWS SDK
|
||||
// (like UpdateGroup with Disabled parameter).
|
||||
func callIAMAPIAuthenticated(_ *testing.T, framework *S3IAMTestFramework, action string, params url.Values) (*http.Response, error) {
|
||||
params.Set("Action", action)
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, TestIAMEndpoint+"/",
|
||||
strings.NewReader(params.Encode()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
|
||||
token, err := framework.generateSTSSessionToken("admin-user", "TestAdminRole", time.Hour, "", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
Transport: &BearerTokenTransport{Token: token},
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
// TestIAMGroupRawAPI tests group operations using raw HTTP IAM API calls,
|
||||
// verifying XML response format for group operations.
|
||||
func TestIAMGroupRawAPI(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
if !isSeaweedFSRunning(t) {
|
||||
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
|
||||
}
|
||||
|
||||
framework := NewS3IAMTestFramework(t)
|
||||
defer framework.Cleanup()
|
||||
|
||||
groupName := "test-raw-api-group"
|
||||
|
||||
t.Run("create_group_raw", func(t *testing.T) {
|
||||
resp, err := callIAMAPIAuthenticated(t, framework, "CreateGroup", url.Values{
|
||||
"GroupName": {groupName},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
|
||||
var createResp CreateGroupResponse
|
||||
err = xml.Unmarshal(body, &createResp)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, groupName, createResp.CreateGroupResult.Group.GroupName)
|
||||
})
|
||||
|
||||
t.Run("list_groups_raw", func(t *testing.T) {
|
||||
resp, err := callIAMAPIAuthenticated(t, framework, "ListGroups", url.Values{})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
|
||||
var listResp ListGroupsResponse
|
||||
err = xml.Unmarshal(body, &listResp)
|
||||
require.NoError(t, err)
|
||||
|
||||
found := false
|
||||
for _, g := range listResp.ListGroupsResult.Groups {
|
||||
if g.GroupName == groupName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Created group should appear in raw ListGroups")
|
||||
})
|
||||
|
||||
t.Run("delete_group_raw", func(t *testing.T) {
|
||||
resp, err := callIAMAPIAuthenticated(t, framework, "DeleteGroup", url.Values{
|
||||
"GroupName": {groupName},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode)
|
||||
})
|
||||
}
|
||||
|
||||
// createS3Client creates an S3 client with static credentials
|
||||
func createS3Client(t *testing.T, accessKey, secretKey string) *s3.S3 {
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String("us-east-1"),
|
||||
Endpoint: aws.String(TestS3Endpoint),
|
||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
|
||||
DisableSSL: aws.Bool(true),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return s3.New(sess)
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
package example
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
v1credentials "github.com/aws/aws-sdk-go/aws/credentials"
|
||||
v1signer "github.com/aws/aws-sdk-go/aws/signer/v4"
|
||||
v1s3 "github.com/aws/aws-sdk-go/service/s3"
|
||||
v2aws "github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
v2s3 "github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newS3V2Client creates an AWS SDK v2 S3 client from the test cluster.
|
||||
func newS3V2Client(cluster *TestCluster) *v2s3.Client {
|
||||
return v2s3.New(v2s3.Options{
|
||||
Region: testRegion,
|
||||
BaseEndpoint: v2aws.String(cluster.s3Endpoint),
|
||||
Credentials: v2aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, "")),
|
||||
UsePathStyle: true,
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetObjectAttributes(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
t.Run("Basic", func(t *testing.T) {
|
||||
testGetObjectAttributesBasic(t, cluster)
|
||||
})
|
||||
t.Run("MultipartObject", func(t *testing.T) {
|
||||
testGetObjectAttributesMultipart(t, cluster)
|
||||
})
|
||||
t.Run("SelectiveAttributes", func(t *testing.T) {
|
||||
testGetObjectAttributesSelective(t, cluster)
|
||||
})
|
||||
t.Run("InvalidAttribute", func(t *testing.T) {
|
||||
testGetObjectAttributesInvalid(t, cluster)
|
||||
})
|
||||
t.Run("NonExistentObject", func(t *testing.T) {
|
||||
testGetObjectAttributesNotFound(t, cluster)
|
||||
})
|
||||
t.Run("VersionedObject", func(t *testing.T) {
|
||||
testGetObjectAttributesVersioned(t, cluster)
|
||||
})
|
||||
t.Run("ConditionalHeaders", func(t *testing.T) {
|
||||
testGetObjectAttributesConditionalHeaders(t, cluster)
|
||||
})
|
||||
t.Run("VersionedConditionalHeaders", func(t *testing.T) {
|
||||
testGetObjectAttributesVersionedConditionalHeaders(t, cluster)
|
||||
})
|
||||
}
|
||||
|
||||
func testGetObjectAttributesBasic(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-basic-")
|
||||
objectKey := "test-object.txt"
|
||||
objectData := "Hello, GetObjectAttributes!"
|
||||
|
||||
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte(objectData)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
client := newS3V2Client(cluster)
|
||||
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String(objectKey),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesEtag,
|
||||
types.ObjectAttributesStorageClass,
|
||||
types.ObjectAttributesObjectSize,
|
||||
types.ObjectAttributesObjectParts,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// ETag should be present and non-empty
|
||||
require.NotNil(t, resp.ETag)
|
||||
assert.NotEmpty(t, *resp.ETag)
|
||||
assert.False(t, strings.Contains(*resp.ETag, `"`), "ETag in XML body should not have quotes")
|
||||
|
||||
// ObjectSize should match
|
||||
require.NotNil(t, resp.ObjectSize)
|
||||
assert.Equal(t, int64(len(objectData)), *resp.ObjectSize)
|
||||
|
||||
// StorageClass should be STANDARD (default)
|
||||
assert.Equal(t, "STANDARD", string(resp.StorageClass))
|
||||
|
||||
// ObjectParts should be nil for non-multipart objects
|
||||
assert.Nil(t, resp.ObjectParts)
|
||||
|
||||
// LastModified header should be present
|
||||
assert.NotNil(t, resp.LastModified)
|
||||
|
||||
t.Logf("Basic GetObjectAttributes passed: ETag=%s, Size=%d, StorageClass=%s",
|
||||
*resp.ETag, *resp.ObjectSize, resp.StorageClass)
|
||||
}
|
||||
|
||||
func testGetObjectAttributesMultipart(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-mp-")
|
||||
objectKey := "test-multipart.bin"
|
||||
|
||||
// Create a 2-part multipart upload
|
||||
part1Data := bytes.Repeat([]byte("A"), 5*1024*1024) // 5MB (minimum part size)
|
||||
part2Data := bytes.Repeat([]byte("B"), 3*1024*1024) // 3MB
|
||||
|
||||
initResp, err := cluster.s3Client.CreateMultipartUpload(&v1s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
uploadID := initResp.UploadId
|
||||
|
||||
part1Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
PartNumber: aws.Int64(1),
|
||||
UploadId: uploadID,
|
||||
Body: bytes.NewReader(part1Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
part2Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
PartNumber: aws.Int64(2),
|
||||
UploadId: uploadID,
|
||||
Body: bytes.NewReader(part2Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = cluster.s3Client.CompleteMultipartUpload(&v1s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadID,
|
||||
MultipartUpload: &v1s3.CompletedMultipartUpload{
|
||||
Parts: []*v1s3.CompletedPart{
|
||||
{ETag: part1Resp.ETag, PartNumber: aws.Int64(1)},
|
||||
{ETag: part2Resp.ETag, PartNumber: aws.Int64(2)},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait briefly for metadata to settle
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
client := newS3V2Client(cluster)
|
||||
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String(objectKey),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesObjectParts,
|
||||
types.ObjectAttributesObjectSize,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NotNil(t, resp.ObjectSize)
|
||||
assert.Equal(t, int64(len(part1Data)+len(part2Data)), *resp.ObjectSize)
|
||||
|
||||
require.NotNil(t, resp.ObjectParts, "ObjectParts should be present for multipart objects")
|
||||
assert.Equal(t, int32(2), *resp.ObjectParts.TotalPartsCount)
|
||||
require.Len(t, resp.ObjectParts.Parts, 2)
|
||||
assert.Equal(t, int32(1), *resp.ObjectParts.Parts[0].PartNumber)
|
||||
assert.Equal(t, int64(len(part1Data)), *resp.ObjectParts.Parts[0].Size)
|
||||
assert.Equal(t, int32(2), *resp.ObjectParts.Parts[1].PartNumber)
|
||||
assert.Equal(t, int64(len(part2Data)), *resp.ObjectParts.Parts[1].Size)
|
||||
|
||||
// Test pagination: MaxParts=1
|
||||
resp2, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String(objectKey),
|
||||
MaxParts: v2aws.Int32(1),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesObjectParts,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp2.ObjectParts)
|
||||
assert.Len(t, resp2.ObjectParts.Parts, 1)
|
||||
assert.True(t, *resp2.ObjectParts.IsTruncated)
|
||||
assert.Equal(t, int32(2), *resp2.ObjectParts.TotalPartsCount)
|
||||
|
||||
t.Logf("Multipart GetObjectAttributes passed: %d parts, total size %d",
|
||||
*resp.ObjectParts.TotalPartsCount, *resp.ObjectSize)
|
||||
}
|
||||
|
||||
func testGetObjectAttributesSelective(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-sel-")
|
||||
objectKey := "test-selective.txt"
|
||||
objectData := "Selective attributes test"
|
||||
|
||||
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte(objectData)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
client := newS3V2Client(cluster)
|
||||
|
||||
// Request only ETag
|
||||
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String(objectKey),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesEtag,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp.ETag)
|
||||
assert.NotEmpty(t, *resp.ETag)
|
||||
assert.Nil(t, resp.ObjectSize, "ObjectSize should not be present when not requested")
|
||||
assert.Empty(t, string(resp.StorageClass), "StorageClass should not be present when not requested")
|
||||
assert.Nil(t, resp.ObjectParts, "ObjectParts should not be present when not requested")
|
||||
|
||||
t.Logf("Selective GetObjectAttributes passed: ETag=%s", *resp.ETag)
|
||||
}
|
||||
|
||||
func testGetObjectAttributesInvalid(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-inv-")
|
||||
objectKey := "test-object.txt"
|
||||
|
||||
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte("test")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Use raw HTTP to send an invalid attribute name since the SDK validates
|
||||
reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Object-Attributes", "InvalidAttr")
|
||||
|
||||
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
|
||||
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
|
||||
require.NoError(t, err)
|
||||
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
|
||||
assert.Equal(t, 400, resp.StatusCode)
|
||||
t.Logf("Invalid attribute test passed: got %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
func testGetObjectAttributesNotFound(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-nf-")
|
||||
|
||||
client := newS3V2Client(cluster)
|
||||
_, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("nonexistent-key"),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesEtag,
|
||||
},
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "NoSuchKey")
|
||||
|
||||
t.Logf("NotFound GetObjectAttributes passed")
|
||||
}
|
||||
|
||||
func testGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster) {
|
||||
client := newS3V2Client(cluster)
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-ver-")
|
||||
|
||||
// Enable versioning
|
||||
_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusEnabled,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
// Put two versions of the same object
|
||||
v1Data := "version 1 content"
|
||||
putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("versioned-key"),
|
||||
Body: strings.NewReader(v1Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp1.VersionId)
|
||||
versionId1 := *putResp1.VersionId
|
||||
|
||||
v2Data := "version 2 content - longer"
|
||||
putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("versioned-key"),
|
||||
Body: strings.NewReader(v2Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp2.VersionId)
|
||||
versionId2 := *putResp2.VersionId
|
||||
|
||||
assert.NotEqual(t, versionId1, versionId2, "versions should differ")
|
||||
|
||||
// GetObjectAttributes for latest version (v2)
|
||||
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("versioned-key"),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesObjectSize,
|
||||
types.ObjectAttributesEtag,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp.ObjectSize)
|
||||
assert.Equal(t, int64(len(v2Data)), *resp.ObjectSize)
|
||||
require.NotNil(t, resp.VersionId)
|
||||
assert.Equal(t, versionId2, *resp.VersionId)
|
||||
|
||||
// GetObjectAttributes for specific older version (v1)
|
||||
resp1, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("versioned-key"),
|
||||
VersionId: v2aws.String(versionId1),
|
||||
ObjectAttributes: []types.ObjectAttributes{
|
||||
types.ObjectAttributesObjectSize,
|
||||
types.ObjectAttributesEtag,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, resp1.ObjectSize)
|
||||
assert.Equal(t, int64(len(v1Data)), *resp1.ObjectSize)
|
||||
require.NotNil(t, resp1.VersionId)
|
||||
assert.Equal(t, versionId1, *resp1.VersionId)
|
||||
|
||||
t.Logf("Versioned GetObjectAttributes passed: v1 size=%d (id=%s), v2 size=%d (id=%s)",
|
||||
*resp1.ObjectSize, versionId1, *resp.ObjectSize, versionId2)
|
||||
}
|
||||
|
||||
// signedGetObjectAttributes creates a signed GET request for ?attributes with custom headers.
|
||||
func signedGetObjectAttributes(t *testing.T, cluster *TestCluster, bucketName, objectKey string, extraHeaders map[string]string) *http.Response {
|
||||
reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
|
||||
for k, v := range extraHeaders {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
|
||||
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
|
||||
require.NoError(t, err)
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
return resp
|
||||
}
|
||||
|
||||
func testGetObjectAttributesConditionalHeaders(t *testing.T, cluster *TestCluster) {
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-cond-")
|
||||
objectKey := "cond-test.txt"
|
||||
|
||||
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader([]byte("conditional headers test")),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Get the ETag and Last-Modified for the object
|
||||
headResp, err := cluster.s3Client.HeadObject(&v1s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
etag := aws.StringValue(headResp.ETag)
|
||||
lastModified := headResp.LastModified
|
||||
require.NotNil(t, lastModified)
|
||||
|
||||
pastDate := lastModified.Add(-1 * time.Hour).UTC().Format(http.TimeFormat)
|
||||
futureDate := lastModified.Add(1 * time.Hour).UTC().Format(http.TimeFormat)
|
||||
|
||||
// RFC 7232: If-Match true + If-Unmodified-Since false => 200 OK
|
||||
// If-Unmodified-Since is ignored when If-Match is present
|
||||
t.Run("IfMatch_true_IfUnmodifiedSince_false", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
|
||||
"If-Match": etag,
|
||||
"If-Unmodified-Since": pastDate, // object was modified after this => false
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 200, resp.StatusCode,
|
||||
"If-Match=true should return 200 even when If-Unmodified-Since=false (RFC 7232 Section 3.4)")
|
||||
})
|
||||
|
||||
// RFC 7232: If-None-Match false + If-Modified-Since true => 304 Not Modified
|
||||
// If-Modified-Since is ignored when If-None-Match is present
|
||||
t.Run("IfNoneMatch_false_IfModifiedSince_true", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
|
||||
"If-None-Match": etag,
|
||||
"If-Modified-Since": pastDate, // object was modified after this => true
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 304, resp.StatusCode,
|
||||
"If-None-Match=false (ETag match) should return 304 even when If-Modified-Since=true (RFC 7232 Section 3.3)")
|
||||
})
|
||||
|
||||
// If-Match succeeds, If-Unmodified-Since also succeeds => 200
|
||||
t.Run("IfMatch_true_IfUnmodifiedSince_true", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
|
||||
"If-Match": etag,
|
||||
"If-Unmodified-Since": futureDate,
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 200, resp.StatusCode)
|
||||
})
|
||||
|
||||
// If-None-Match passes (ETag differs), If-Modified-Since ignored => 200
|
||||
// Per RFC 7232, If-Modified-Since is ignored when If-None-Match is present
|
||||
t.Run("IfNoneMatch_true_IfModifiedSince_ignored", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
|
||||
"If-None-Match": `"nonexistent-etag"`,
|
||||
"If-Modified-Since": futureDate, // would fail alone, but is ignored
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 200, resp.StatusCode,
|
||||
"If-None-Match=true means If-Modified-Since is ignored, should return 200 (RFC 7232 Section 3.3)")
|
||||
})
|
||||
|
||||
// If-Match fails => 412 regardless of If-Unmodified-Since
|
||||
t.Run("IfMatch_false", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
|
||||
"If-Match": `"wrong-etag"`,
|
||||
"If-Unmodified-Since": futureDate,
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 412, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Logf("Conditional headers tests passed")
|
||||
}
|
||||
|
||||
// signedGetObjectAttributesVersioned creates a signed GET request for ?attributes&versionId=... with custom headers.
|
||||
func signedGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster, bucketName, objectKey, versionId string, extraHeaders map[string]string) *http.Response {
|
||||
reqURL := fmt.Sprintf("%s/%s/%s?attributes&versionId=%s", cluster.s3Endpoint, bucketName, objectKey, versionId)
|
||||
req, err := http.NewRequest("GET", reqURL, nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
|
||||
for k, v := range extraHeaders {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
|
||||
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
|
||||
require.NoError(t, err)
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
return resp
|
||||
}
|
||||
|
||||
func testGetObjectAttributesVersionedConditionalHeaders(t *testing.T, cluster *TestCluster) {
|
||||
client := newS3V2Client(cluster)
|
||||
bucketName := createTestBucket(t, cluster, "test-goa-vcond-")
|
||||
|
||||
// Enable versioning
|
||||
_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusEnabled,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
||||
// Put two versions with different content (different ETags)
|
||||
v1Data := "version 1 - original"
|
||||
putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("vcond-key"),
|
||||
Body: strings.NewReader(v1Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp1.VersionId)
|
||||
vid1 := *putResp1.VersionId
|
||||
|
||||
v2Data := "version 2 - updated content"
|
||||
putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("vcond-key"),
|
||||
Body: strings.NewReader(v2Data),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, putResp2.VersionId)
|
||||
vid2 := *putResp2.VersionId
|
||||
|
||||
// Get ETags for each version
|
||||
headV1, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("vcond-key"),
|
||||
VersionId: v2aws.String(vid1),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
etagV1 := *headV1.ETag
|
||||
|
||||
headV2, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
|
||||
Bucket: v2aws.String(bucketName),
|
||||
Key: v2aws.String("vcond-key"),
|
||||
VersionId: v2aws.String(vid2),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
etagV2 := *headV2.ETag
|
||||
require.NotEqual(t, etagV1, etagV2, "versions should have different ETags")
|
||||
|
||||
// If-Match with v1's ETag + versionId=v1 => 200
|
||||
// Before the fix, this would fail with 412 because conditional headers
|
||||
// were evaluated against the latest version (v2) whose ETag differs
|
||||
t.Run("IfMatch_v1_etag_versionId_v1", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
|
||||
"If-Match": etagV1,
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 200, resp.StatusCode,
|
||||
"If-Match with v1 ETag targeting versionId=v1 should return 200")
|
||||
})
|
||||
|
||||
// If-Match with v2's ETag + versionId=v1 => 412
|
||||
// The ETag doesn't match v1, so this should fail
|
||||
t.Run("IfMatch_v2_etag_versionId_v1", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
|
||||
"If-Match": etagV2,
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 412, resp.StatusCode,
|
||||
"If-Match with v2 ETag targeting versionId=v1 should return 412")
|
||||
})
|
||||
|
||||
// If-None-Match with v1's ETag + versionId=v1 => 304
|
||||
t.Run("IfNoneMatch_v1_etag_versionId_v1", func(t *testing.T) {
|
||||
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
|
||||
"If-None-Match": etagV1,
|
||||
})
|
||||
defer resp.Body.Close()
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
assert.Equal(t, 304, resp.StatusCode,
|
||||
"If-None-Match with v1 ETag targeting versionId=v1 should return 304")
|
||||
})
|
||||
|
||||
t.Logf("Versioned conditional headers tests passed: vid1=%s, vid2=%s", vid1, vid2)
|
||||
}
|
||||
@@ -422,6 +422,218 @@ func TestS3MultipartOperationsInheritPutObjectPermissions(t *testing.T) {
|
||||
require.Equal(t, 0, len(listUploadsOut.Uploads))
|
||||
}
|
||||
|
||||
// TestS3IAMManagedPolicyLifecycle is an end-to-end integration test covering the
|
||||
// user-reported use case in https://github.com/seaweedfs/seaweedfs/issues/8506
|
||||
// where managed policy operations (GetPolicy, ListPolicies, DeletePolicy,
|
||||
// AttachUserPolicy, DetachUserPolicy) returned 500 errors.
|
||||
func TestS3IAMManagedPolicyLifecycle(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
iamClient := newIAMClient(t, cluster.s3Endpoint)
|
||||
|
||||
// Step 1: Create a user (this already worked per the issue)
|
||||
userName := uniqueName("lifecycle-user")
|
||||
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err, "CreateUser should succeed")
|
||||
|
||||
// Step 2: Create a managed policy via IAM API
|
||||
policyName := uniqueName("lifecycle-policy")
|
||||
policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
|
||||
policyDoc := `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject"],
|
||||
"Resource": "arn:aws:s3:::*"
|
||||
}]
|
||||
}`
|
||||
createOut, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(policyDoc),
|
||||
})
|
||||
require.NoError(t, err, "CreatePolicy should succeed")
|
||||
require.NotNil(t, createOut.Policy)
|
||||
require.Equal(t, policyName, *createOut.Policy.PolicyName)
|
||||
|
||||
// Step 3: ListPolicies — should include the created policy (was returning 500)
|
||||
listOut, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
|
||||
require.NoError(t, err, "ListPolicies should succeed (was returning 500)")
|
||||
require.True(t, managedPolicyContains(listOut.Policies, policyName),
|
||||
"ListPolicies should contain the newly created policy")
|
||||
|
||||
// Step 4: GetPolicy by ARN — should return the policy (was returning 500)
|
||||
getOut, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
|
||||
require.NoError(t, err, "GetPolicy should succeed (was returning 500)")
|
||||
require.NotNil(t, getOut.Policy)
|
||||
require.Equal(t, policyName, *getOut.Policy.PolicyName)
|
||||
require.Equal(t, policyArn, *getOut.Policy.Arn)
|
||||
|
||||
// Step 5: AttachUserPolicy — should succeed (was returning 500)
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(policyArn),
|
||||
})
|
||||
require.NoError(t, err, "AttachUserPolicy should succeed (was returning 500)")
|
||||
|
||||
// Step 6: ListAttachedUserPolicies — verify the policy is attached
|
||||
attachedOut, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err, "ListAttachedUserPolicies should succeed")
|
||||
require.True(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
|
||||
"Policy should appear in user's attached policies")
|
||||
|
||||
// Step 7: Idempotent re-attach should not fail
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(policyArn),
|
||||
})
|
||||
require.NoError(t, err, "Re-attaching same policy should be idempotent")
|
||||
|
||||
// Step 8: DeletePolicy while attached — should fail with DeleteConflict (AWS behavior)
|
||||
_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
|
||||
require.Error(t, err, "DeletePolicy should fail while policy is attached")
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeDeleteConflictException, awsErr.Code(),
|
||||
"Should return DeleteConflict when deleting attached policy")
|
||||
|
||||
// Step 9: DetachUserPolicy
|
||||
_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(policyArn),
|
||||
})
|
||||
require.NoError(t, err, "DetachUserPolicy should succeed")
|
||||
|
||||
// Verify detached
|
||||
attachedOut, err = iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
|
||||
UserName: aws.String(userName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.False(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
|
||||
"Policy should no longer appear in user's attached policies after detach")
|
||||
|
||||
// Step 10: DeletePolicy — should now succeed (was returning XML parsing error)
|
||||
_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
|
||||
require.NoError(t, err, "DeletePolicy should succeed after detach (was returning XML parsing error)")
|
||||
|
||||
// Step 11: Verify the policy is gone
|
||||
listOut, err = iamClient.ListPolicies(&iam.ListPoliciesInput{})
|
||||
require.NoError(t, err)
|
||||
require.False(t, managedPolicyContains(listOut.Policies, policyName),
|
||||
"Deleted policy should not appear in ListPolicies")
|
||||
|
||||
_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
|
||||
require.Error(t, err, "GetPolicy should fail for deleted policy")
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
}
|
||||
|
||||
// TestS3IAMManagedPolicyErrorCases covers error cases from the user-reported issue:
|
||||
// invalid ARNs, missing policies, and missing users.
|
||||
func TestS3IAMManagedPolicyErrorCases(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
iamClient := newIAMClient(t, cluster.s3Endpoint)
|
||||
|
||||
t.Run("GetPolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
|
||||
_, err := iamClient.GetPolicy(&iam.GetPolicyInput{
|
||||
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("DeletePolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
|
||||
_, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{
|
||||
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("AttachUserPolicy with nonexistent policy returns NoSuchEntity", func(t *testing.T) {
|
||||
userName := uniqueName("err-user")
|
||||
_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("AttachUserPolicy with nonexistent user returns NoSuchEntity", func(t *testing.T) {
|
||||
policyName := uniqueName("err-policy")
|
||||
_, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
|
||||
UserName: aws.String("nonexistent-user"),
|
||||
PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("DetachUserPolicy that is not attached returns NoSuchEntity", func(t *testing.T) {
|
||||
userName := uniqueName("detach-user")
|
||||
_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
|
||||
require.NoError(t, err)
|
||||
|
||||
policyName := uniqueName("detach-policy")
|
||||
_, err = iamClient.CreatePolicy(&iam.CreatePolicyInput{
|
||||
PolicyName: aws.String(policyName),
|
||||
PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
|
||||
UserName: aws.String(userName),
|
||||
PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
|
||||
t.Run("ListAttachedUserPolicies for nonexistent user returns NoSuchEntity", func(t *testing.T) {
|
||||
_, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
|
||||
UserName: aws.String("nonexistent-user"),
|
||||
})
|
||||
require.Error(t, err)
|
||||
var awsErr awserr.Error
|
||||
require.True(t, errors.As(err, &awsErr))
|
||||
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
|
||||
})
|
||||
}
|
||||
|
||||
func execShell(t *testing.T, weedCmd, master, filer, shellCmd string) string {
|
||||
// weed shell -master=... -filer=...
|
||||
args := []string{"shell", "-master=" + master, "-filer=" + filer}
|
||||
|
||||
@@ -1,15 +1,32 @@
|
||||
package lakekeeper
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/aws-sdk-go-v2/service/sts"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
type TestEnvironment struct {
|
||||
@@ -31,15 +48,19 @@ type TestEnvironment struct {
|
||||
secretKey string
|
||||
}
|
||||
|
||||
const (
|
||||
lakekeeperRegion = "us-east-1"
|
||||
lakekeeperRoleArn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
|
||||
lakekeeperSessionName = "lakekeeper-session"
|
||||
)
|
||||
|
||||
type lakekeeperSession struct {
|
||||
endpoint string
|
||||
region string
|
||||
creds aws.Credentials
|
||||
}
|
||||
|
||||
func TestLakekeeperIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
if !testutil.HasDocker() {
|
||||
t.Skip("Docker not available, skipping Lakekeeper integration test")
|
||||
}
|
||||
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
@@ -51,6 +72,18 @@ func TestLakekeeperIntegration(t *testing.T) {
|
||||
runLakekeeperRepro(t, env)
|
||||
}
|
||||
|
||||
func TestLakekeeperTableBucketIntegration(t *testing.T) {
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
fmt.Printf(">>> Starting SeaweedFS with Lakekeeper configuration...\n")
|
||||
env.StartSeaweedFS(t)
|
||||
fmt.Printf(">>> SeaweedFS started.\n")
|
||||
|
||||
// Run python script in docker to test STS and S3 Tables operations
|
||||
runLakekeeperTableBucketRepro(t, env)
|
||||
}
|
||||
|
||||
func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
t.Helper()
|
||||
|
||||
@@ -218,124 +251,439 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
|
||||
func runLakekeeperRepro(t *testing.T, env *TestEnvironment) {
|
||||
t.Helper()
|
||||
|
||||
scriptContent := fmt.Sprintf(`
|
||||
import boto3
|
||||
import botocore.config
|
||||
import botocore
|
||||
from botocore.exceptions import ClientError
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
# Enable botocore debug logging to see signature calculation
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
botocore.session.get_session().set_debug_logger()
|
||||
|
||||
print("Starting Lakekeeper repro test...")
|
||||
|
||||
endpoint_url = "http://host.docker.internal:%d"
|
||||
access_key = "%s"
|
||||
secret_key = "%s"
|
||||
region = "us-east-1"
|
||||
|
||||
print(f"Connecting to {endpoint_url}")
|
||||
|
||||
try:
|
||||
config = botocore.config.Config(
|
||||
retries={'max_attempts': 3}
|
||||
)
|
||||
sts = boto3.client(
|
||||
'sts',
|
||||
endpoint_url=endpoint_url,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name=region,
|
||||
config=config
|
||||
)
|
||||
|
||||
role_arn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
|
||||
session_name = "lakekeeper-session"
|
||||
|
||||
print(f"Calling AssumeRole on {role_arn} with POST body...")
|
||||
|
||||
# Standard boto3 call sends parameters in POST body
|
||||
response = sts.assume_role(
|
||||
RoleArn=role_arn,
|
||||
RoleSessionName=session_name
|
||||
)
|
||||
|
||||
creds = response['Credentials']
|
||||
access_key_id = creds['AccessKeyId']
|
||||
secret_access_key = creds['SecretAccessKey']
|
||||
session_token = creds['SessionToken']
|
||||
|
||||
print(f"Success! Got credentials with prefix: {access_key_id[:4]}")
|
||||
|
||||
if not access_key_id.startswith("ASIA"):
|
||||
print(f"FAILED: Expected ASIA prefix, got {access_key_id}")
|
||||
sys.exit(1)
|
||||
|
||||
print("Verifying S3 operations with vended credentials...")
|
||||
s3 = boto3.client(
|
||||
's3',
|
||||
endpoint_url=endpoint_url,
|
||||
aws_access_key_id=access_key_id,
|
||||
aws_secret_access_key=secret_access_key,
|
||||
aws_session_token=session_token,
|
||||
region_name=region,
|
||||
config=config
|
||||
)
|
||||
|
||||
bucket = "lakekeeper-vended-bucket"
|
||||
print(f"Creating bucket {bucket}...")
|
||||
s3.create_bucket(Bucket=bucket)
|
||||
|
||||
print("Listing buckets...")
|
||||
response = s3.list_buckets()
|
||||
buckets = [b['Name'] for b in response['Buckets']]
|
||||
print(f"Found buckets: {buckets}")
|
||||
|
||||
if bucket not in buckets:
|
||||
print(f"FAILED: Bucket {bucket} not found in list")
|
||||
sys.exit(1)
|
||||
|
||||
print("SUCCESS: Lakekeeper flow verified!")
|
||||
sys.exit(0)
|
||||
|
||||
except Exception as e:
|
||||
print(f"FAILED: {e}")
|
||||
# Print more details if it is a ClientError
|
||||
if hasattr(e, 'response'):
|
||||
print(f"Response: {e.response}")
|
||||
sys.exit(1)
|
||||
`, env.s3Port, env.accessKey, env.secretKey)
|
||||
|
||||
scriptPath := filepath.Join(env.dataDir, "lakekeeper_repro.py")
|
||||
if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
|
||||
t.Fatalf("Failed to write python script: %v", err)
|
||||
}
|
||||
|
||||
containerName := "seaweed-lakekeeper-client-" + fmt.Sprintf("%d", time.Now().UnixNano())
|
||||
|
||||
// Create a context with timeout for the docker run command
|
||||
dockerCtx, dockerCancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer dockerCancel()
|
||||
|
||||
cmd := exec.CommandContext(dockerCtx, "docker", "run", "--rm",
|
||||
"--name", containerName,
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-v", fmt.Sprintf("%s:/work", env.dataDir),
|
||||
"python:3",
|
||||
"/bin/bash", "-c", "pip install boto3 && python /work/lakekeeper_repro.py",
|
||||
)
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
session, err := newLakekeeperSession(ctx, env)
|
||||
if err != nil {
|
||||
if dockerCtx.Err() == context.DeadlineExceeded {
|
||||
t.Fatalf("Lakekeeper repro client timed out after 5 minutes\nOutput:\n%s", string(output))
|
||||
}
|
||||
t.Fatalf("Lakekeeper repro client failed: %v\nOutput:\n%s", err, string(output))
|
||||
t.Fatalf("AssumeRole failed: %v", err)
|
||||
}
|
||||
t.Logf("Lakekeeper repro client output:\n%s", string(output))
|
||||
|
||||
s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
|
||||
if err != nil {
|
||||
t.Fatalf("Create S3 client failed: %v", err)
|
||||
}
|
||||
|
||||
bucketName := fmt.Sprintf("lakekeeper-vended-bucket-%d", time.Now().UnixNano())
|
||||
if _, err := s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateBucket failed: %v", err)
|
||||
}
|
||||
|
||||
bucketCreated := true
|
||||
defer func() {
|
||||
if !bucketCreated {
|
||||
return
|
||||
}
|
||||
_, _ = s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
}()
|
||||
|
||||
listResp, err := s3Client.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
if err != nil {
|
||||
t.Fatalf("ListBuckets failed: %v", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, bucket := range listResp.Buckets {
|
||||
if aws.ToString(bucket.Name) == bucketName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("Bucket %s not found in list", bucketName)
|
||||
}
|
||||
|
||||
if _, err := s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
}); err != nil {
|
||||
t.Fatalf("DeleteBucket failed: %v", err)
|
||||
}
|
||||
bucketCreated = false
|
||||
}
|
||||
|
||||
func runLakekeeperTableBucketRepro(t *testing.T, env *TestEnvironment) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
session, err := newLakekeeperSession(ctx, env)
|
||||
if err != nil {
|
||||
t.Fatalf("AssumeRole failed: %v", err)
|
||||
}
|
||||
|
||||
client := newS3TablesClient(session.endpoint, session.region, session.creds)
|
||||
bucketName := fmt.Sprintf("lakekeeper-table-bucket-%d", time.Now().UnixNano())
|
||||
bucketARN, err := client.CreateTableBucket(ctx, bucketName)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateTableBucket failed: %v", err)
|
||||
}
|
||||
|
||||
bucketDeleted := false
|
||||
namespaceCreated := false
|
||||
tableCreated := false
|
||||
namespaceName := fmt.Sprintf("lakekeeper_ns_%d", time.Now().UnixNano())
|
||||
tableName := fmt.Sprintf("lakekeeper_table_%d", time.Now().UnixNano())
|
||||
defer func() {
|
||||
if tableCreated {
|
||||
if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
|
||||
t.Logf("Failed to delete table: %v", err)
|
||||
}
|
||||
}
|
||||
if namespaceCreated {
|
||||
if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
|
||||
t.Logf("Failed to delete namespace: %v", err)
|
||||
}
|
||||
}
|
||||
if bucketDeleted {
|
||||
return
|
||||
}
|
||||
if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
|
||||
t.Logf("Failed to delete table bucket: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
buckets, err := client.ListTableBuckets(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("ListTableBuckets failed: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, b := range buckets {
|
||||
if b.Name == bucketName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("Created table bucket %s not found in list", bucketName)
|
||||
}
|
||||
|
||||
if _, err := client.GetTableBucket(ctx, bucketARN); err != nil {
|
||||
t.Fatalf("GetTableBucket failed: %v", err)
|
||||
}
|
||||
|
||||
if err := client.CreateNamespace(ctx, bucketARN, namespaceName); err != nil {
|
||||
t.Fatalf("CreateNamespace failed: %v", err)
|
||||
}
|
||||
namespaceCreated = true
|
||||
|
||||
if err := client.CreateTable(ctx, bucketARN, namespaceName, tableName); err != nil {
|
||||
t.Fatalf("CreateTable failed: %v", err)
|
||||
}
|
||||
tableCreated = true
|
||||
|
||||
s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
|
||||
if err != nil {
|
||||
t.Fatalf("Create S3 client failed: %v", err)
|
||||
}
|
||||
|
||||
objectKey := fmt.Sprintf("%s/%s/data/part-%d.parquet", namespaceName, tableName, time.Now().UnixNano())
|
||||
createResp, err := s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateMultipartUpload failed: %v", err)
|
||||
}
|
||||
uploadID := aws.ToString(createResp.UploadId)
|
||||
multipartCompleted := false
|
||||
defer func() {
|
||||
if uploadID == "" || multipartCompleted {
|
||||
return
|
||||
}
|
||||
_, _ = s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
})
|
||||
}()
|
||||
|
||||
partSize := 5 * 1024 * 1024
|
||||
part1 := bytes.Repeat([]byte("a"), partSize)
|
||||
part2 := bytes.Repeat([]byte("b"), 1024*1024)
|
||||
|
||||
part1Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader(part1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UploadPart 1 failed: %v", err)
|
||||
}
|
||||
|
||||
part2Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
PartNumber: aws.Int32(2),
|
||||
Body: bytes.NewReader(part2),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UploadPart 2 failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
MultipartUpload: &s3types.CompletedMultipartUpload{
|
||||
Parts: []s3types.CompletedPart{
|
||||
{
|
||||
ETag: part1Resp.ETag,
|
||||
PartNumber: aws.Int32(1),
|
||||
},
|
||||
{
|
||||
ETag: part2Resp.ETag,
|
||||
PartNumber: aws.Int32(2),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteMultipartUpload failed: %v", err)
|
||||
}
|
||||
multipartCompleted = true
|
||||
|
||||
headResp, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("HeadObject after multipart upload failed: %v", err)
|
||||
}
|
||||
expectedSize := int64(len(part1) + len(part2))
|
||||
if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
|
||||
t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
|
||||
}
|
||||
|
||||
if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
|
||||
t.Fatalf("DeleteTable failed: %v", err)
|
||||
}
|
||||
tableCreated = false
|
||||
|
||||
if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
|
||||
t.Fatalf("DeleteNamespace failed: %v", err)
|
||||
}
|
||||
namespaceCreated = false
|
||||
|
||||
if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
|
||||
t.Fatalf("DeleteTableBucket failed: %v", err)
|
||||
}
|
||||
bucketDeleted = true
|
||||
|
||||
if _, err := client.GetTableBucket(ctx, bucketARN); err == nil {
|
||||
t.Fatalf("expected GetTableBucket to fail after deletion")
|
||||
}
|
||||
}
|
||||
|
||||
type s3TablesClient struct {
|
||||
endpoint string
|
||||
region string
|
||||
creds aws.Credentials
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func newS3TablesClient(endpoint, region string, creds aws.Credentials) *s3TablesClient {
|
||||
return &s3TablesClient{
|
||||
endpoint: endpoint,
|
||||
region: region,
|
||||
creds: creds,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) CreateTableBucket(ctx context.Context, name string) (string, error) {
|
||||
req := &s3tables.CreateTableBucketRequest{Name: name}
|
||||
var resp s3tables.CreateTableBucketResponse
|
||||
if err := c.doRequest(ctx, "CreateTableBucket", http.MethodPut, "/buckets", req, &resp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return resp.ARN, nil
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) GetTableBucket(ctx context.Context, arn string) (*s3tables.GetTableBucketResponse, error) {
|
||||
path := "/buckets/" + url.PathEscape(arn)
|
||||
var resp s3tables.GetTableBucketResponse
|
||||
if err := c.doRequest(ctx, "GetTableBucket", http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) ListTableBuckets(ctx context.Context) ([]s3tables.TableBucketSummary, error) {
|
||||
var resp s3tables.ListTableBucketsResponse
|
||||
if err := c.doRequest(ctx, "ListTableBuckets", http.MethodGet, "/buckets", nil, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return resp.TableBuckets, nil
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) DeleteTableBucket(ctx context.Context, arn string) error {
|
||||
path := "/buckets/" + url.PathEscape(arn)
|
||||
return c.doRequest(ctx, "DeleteTableBucket", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) CreateNamespace(ctx context.Context, bucketARN, namespace string) error {
|
||||
req := &s3tables.CreateNamespaceRequest{
|
||||
Namespace: []string{namespace},
|
||||
}
|
||||
path := "/namespaces/" + url.PathEscape(bucketARN)
|
||||
return c.doRequest(ctx, "CreateNamespace", http.MethodPut, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) DeleteNamespace(ctx context.Context, bucketARN, namespace string) error {
|
||||
path := "/namespaces/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
|
||||
return c.doRequest(ctx, "DeleteNamespace", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) CreateTable(ctx context.Context, bucketARN, namespace, name string) error {
|
||||
req := &s3tables.CreateTableRequest{
|
||||
Name: name,
|
||||
Format: "ICEBERG",
|
||||
}
|
||||
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
|
||||
return c.doRequest(ctx, "CreateTable", http.MethodPut, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) DeleteTable(ctx context.Context, bucketARN, namespace, name string) error {
|
||||
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace) + "/" + url.PathEscape(name)
|
||||
return c.doRequest(ctx, "DeleteTable", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *s3TablesClient) doRequest(ctx context.Context, operation, method, path string, body interface{}, out interface{}) error {
|
||||
var bodyBytes []byte
|
||||
if body != nil {
|
||||
encoded, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: marshal request: %w", operation, err)
|
||||
}
|
||||
bodyBytes = encoded
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.endpoint+path, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: create request: %w", operation, err)
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
|
||||
}
|
||||
req.Host = req.URL.Host
|
||||
req.Header.Set("Host", req.URL.Host)
|
||||
|
||||
payloadHash := sha256.Sum256(bodyBytes)
|
||||
if err := v4.NewSigner().SignHTTP(ctx, c.creds, req, hex.EncodeToString(payloadHash[:]), "s3tables", c.region, time.Now()); err != nil {
|
||||
return fmt.Errorf("%s: sign request: %w", operation, err)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: request failed: %w", operation, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("%s failed with status %d and could not read response body: %v", operation, resp.StatusCode, readErr)
|
||||
}
|
||||
var errResp s3tables.S3TablesError
|
||||
if jsonErr := json.Unmarshal(bodyBytes, &errResp); jsonErr == nil && (errResp.Type != "" || errResp.Message != "") {
|
||||
return fmt.Errorf("%s failed: %s - %s", operation, errResp.Type, errResp.Message)
|
||||
}
|
||||
return fmt.Errorf("%s failed with status %d: %s", operation, resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
|
||||
}
|
||||
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
|
||||
return fmt.Errorf("%s: decode response: %w", operation, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func assumeRole(ctx context.Context, endpoint, region, accessKey, secretKey, roleArn, sessionName string) (aws.Credentials, error) {
|
||||
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
if service == sts.ServiceID {
|
||||
return aws.Endpoint{
|
||||
URL: endpoint,
|
||||
SigningRegion: region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
|
||||
})
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
|
||||
config.WithEndpointResolverWithOptions(resolver),
|
||||
)
|
||||
if err != nil {
|
||||
return aws.Credentials{}, err
|
||||
}
|
||||
|
||||
client := sts.NewFromConfig(cfg)
|
||||
resp, err := client.AssumeRole(ctx, &sts.AssumeRoleInput{
|
||||
RoleArn: aws.String(roleArn),
|
||||
RoleSessionName: aws.String(sessionName),
|
||||
})
|
||||
if err != nil {
|
||||
return aws.Credentials{}, err
|
||||
}
|
||||
if resp.Credentials == nil {
|
||||
return aws.Credentials{}, fmt.Errorf("missing credentials in AssumeRole response")
|
||||
}
|
||||
return aws.Credentials{
|
||||
AccessKeyID: aws.ToString(resp.Credentials.AccessKeyId),
|
||||
SecretAccessKey: aws.ToString(resp.Credentials.SecretAccessKey),
|
||||
SessionToken: aws.ToString(resp.Credentials.SessionToken),
|
||||
Source: "lakekeeper-sts",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials) (*s3.Client, error) {
|
||||
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
if service == s3.ServiceID {
|
||||
return aws.Endpoint{
|
||||
URL: endpoint,
|
||||
SigningRegion: region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
|
||||
})
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
|
||||
config.WithEndpointResolverWithOptions(resolver),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.UsePathStyle = true
|
||||
}), nil
|
||||
}
|
||||
|
||||
func newLakekeeperSession(ctx context.Context, env *TestEnvironment) (lakekeeperSession, error) {
|
||||
endpoint := fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
|
||||
creds, err := assumeRole(ctx, endpoint, lakekeeperRegion, env.accessKey, env.secretKey, lakekeeperRoleArn, lakekeeperSessionName)
|
||||
if err != nil {
|
||||
return lakekeeperSession{}, err
|
||||
}
|
||||
return lakekeeperSession{
|
||||
endpoint: endpoint,
|
||||
region: lakekeeperRegion,
|
||||
creds: creds,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,377 @@
|
||||
package polaris
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
|
||||
)
|
||||
|
||||
const (
|
||||
polarisImage = "apache/polaris:latest"
|
||||
polarisRealm = "POLARIS"
|
||||
polarisRootClientID = "root"
|
||||
polarisRootClientSecret = "s3cr3t"
|
||||
polarisRegion = "us-east-1"
|
||||
polarisRoleArn = "arn:aws:iam::000000000000:role/PolarisVendedRole"
|
||||
polarisSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test signing key
|
||||
)
|
||||
|
||||
type TestEnvironment struct {
|
||||
seaweedDir string
|
||||
weedBinary string
|
||||
dataDir string
|
||||
bindIP string
|
||||
s3Port int
|
||||
s3GrpcPort int
|
||||
masterPort int
|
||||
masterGrpcPort int
|
||||
filerPort int
|
||||
filerGrpcPort int
|
||||
volumePort int
|
||||
volumeGrpcPort int
|
||||
polarisPort int
|
||||
polarisAdminPort int
|
||||
weedProcess *exec.Cmd
|
||||
weedCancel context.CancelFunc
|
||||
polarisContainer string
|
||||
accessKey string
|
||||
secretKey string
|
||||
}
|
||||
|
||||
func NewTestEnvironment(t *testing.T) *TestEnvironment {
|
||||
t.Helper()
|
||||
|
||||
if !testutil.HasDocker() {
|
||||
t.Skip("Docker is required for Polaris integration tests")
|
||||
}
|
||||
|
||||
wd, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get working directory: %v", err)
|
||||
}
|
||||
|
||||
seaweedDir := wd
|
||||
for i := 0; i < 6; i++ {
|
||||
if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
|
||||
break
|
||||
}
|
||||
seaweedDir = filepath.Dir(seaweedDir)
|
||||
}
|
||||
|
||||
weedBinary := filepath.Join(seaweedDir, "weed", "weed")
|
||||
if _, err := os.Stat(weedBinary); err != nil {
|
||||
weedBinary = "weed"
|
||||
if _, err := exec.LookPath(weedBinary); err != nil {
|
||||
t.Skip("weed binary not found, skipping integration test")
|
||||
}
|
||||
}
|
||||
|
||||
dataDir, err := os.MkdirTemp("", "seaweed-polaris-test-*")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp dir: %v", err)
|
||||
}
|
||||
|
||||
bindIP := testutil.FindBindIP()
|
||||
|
||||
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
|
||||
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
|
||||
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
|
||||
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
|
||||
polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
|
||||
|
||||
return &TestEnvironment{
|
||||
seaweedDir: seaweedDir,
|
||||
weedBinary: weedBinary,
|
||||
dataDir: dataDir,
|
||||
bindIP: bindIP,
|
||||
s3Port: s3Port,
|
||||
s3GrpcPort: s3GrpcPort,
|
||||
masterPort: masterPort,
|
||||
masterGrpcPort: masterGrpcPort,
|
||||
filerPort: filerPort,
|
||||
filerGrpcPort: filerGrpcPort,
|
||||
volumePort: volumePort,
|
||||
volumeGrpcPort: volumeGrpcPort,
|
||||
polarisPort: polarisPort,
|
||||
polarisAdminPort: polarisAdminPort,
|
||||
accessKey: "admin",
|
||||
secretKey: "admin",
|
||||
}
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
iamConfigPath := filepath.Join(env.dataDir, "iam.json")
|
||||
iamConfig := fmt.Sprintf(`{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "%s",
|
||||
"secretKey": "%s"
|
||||
}
|
||||
],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
|
||||
}
|
||||
],
|
||||
"sts": {
|
||||
"tokenDuration": "12h",
|
||||
"maxSessionLength": "24h",
|
||||
"issuer": "seaweedfs-sts",
|
||||
"signingKey": "%s"
|
||||
},
|
||||
"roles": [
|
||||
{
|
||||
"roleName": "PolarisVendedRole",
|
||||
"roleArn": "%s",
|
||||
"trustPolicy": {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "sts:AssumeRole"
|
||||
}
|
||||
]
|
||||
},
|
||||
"attachedPolicies": ["FullAccess"]
|
||||
}
|
||||
],
|
||||
"policies": [
|
||||
{
|
||||
"name": "FullAccess",
|
||||
"document": {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": "*",
|
||||
"Resource": "*"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}`, env.accessKey, env.secretKey, polarisSigningKey, polarisRoleArn)
|
||||
|
||||
if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
|
||||
t.Fatalf("Failed to create IAM config: %v", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
env.weedCancel = cancel
|
||||
|
||||
cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini",
|
||||
"-master.port", fmt.Sprintf("%d", env.masterPort),
|
||||
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
|
||||
"-volume.port", fmt.Sprintf("%d", env.volumePort),
|
||||
"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
|
||||
"-filer.port", fmt.Sprintf("%d", env.filerPort),
|
||||
"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
|
||||
"-s3.port", fmt.Sprintf("%d", env.s3Port),
|
||||
"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
|
||||
"-s3.config", iamConfigPath,
|
||||
"-s3.iam.config", iamConfigPath,
|
||||
"-s3.iam.readOnly=false",
|
||||
"-ip", env.bindIP,
|
||||
"-ip.bind", "0.0.0.0",
|
||||
"-dir", env.dataDir,
|
||||
)
|
||||
cmd.Dir = env.dataDir
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatalf("Failed to start SeaweedFS: %v", err)
|
||||
}
|
||||
env.weedProcess = cmd
|
||||
|
||||
if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
|
||||
t.Fatalf("S3 API failed to become ready")
|
||||
}
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) StartPolaris(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
containerName := fmt.Sprintf("seaweed-polaris-%d", time.Now().UnixNano())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, "docker", "run", "-d", "--rm",
|
||||
"--name", containerName,
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-p", fmt.Sprintf("%d:8181", env.polarisPort),
|
||||
"-p", fmt.Sprintf("%d:8182", env.polarisAdminPort),
|
||||
"-e", fmt.Sprintf("AWS_REGION=%s", polarisRegion),
|
||||
"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", env.accessKey),
|
||||
"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", env.secretKey),
|
||||
"-e", fmt.Sprintf("POLARIS_BOOTSTRAP_CREDENTIALS=%s,%s,%s", polarisRealm, polarisRootClientID, polarisRootClientSecret),
|
||||
"-e", fmt.Sprintf("polaris.realm-context.realms=%s", polarisRealm),
|
||||
"-e", "quarkus.otel.sdk.disabled=true",
|
||||
polarisImage,
|
||||
)
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
t.Fatalf("Timed out waiting for Polaris container: %v\nOutput:\n%s", ctx.Err(), string(output))
|
||||
}
|
||||
t.Fatalf("Failed to start Polaris: %v\nOutput:\n%s", err, string(output))
|
||||
}
|
||||
env.polarisContainer = containerName
|
||||
|
||||
if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/q/health", env.polarisAdminPort), 60*time.Second) {
|
||||
logCtx, logCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer logCancel()
|
||||
logs, _ := exec.CommandContext(logCtx, "docker", "logs", env.polarisContainer).CombinedOutput()
|
||||
t.Fatalf("Polaris failed to become ready\nLogs:\n%s", string(logs))
|
||||
}
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) Cleanup(t *testing.T) {
|
||||
t.Helper()
|
||||
if env.weedCancel != nil {
|
||||
env.weedCancel()
|
||||
}
|
||||
if env.weedProcess != nil {
|
||||
time.Sleep(1 * time.Second)
|
||||
_ = env.weedProcess.Wait()
|
||||
}
|
||||
if env.polarisContainer != "" {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
_ = exec.CommandContext(ctx, "docker", "rm", "-f", env.polarisContainer).Run()
|
||||
}
|
||||
if env.dataDir != "" {
|
||||
_ = os.RemoveAll(env.dataDir)
|
||||
}
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) polarisEndpoint() string {
|
||||
return fmt.Sprintf("http://127.0.0.1:%d", env.polarisPort)
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) s3Endpoint() string {
|
||||
return fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
|
||||
}
|
||||
|
||||
func (env *TestEnvironment) s3InternalEndpoint() string {
|
||||
return fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
|
||||
}
|
||||
|
||||
type polarisHTTPClient struct {
|
||||
baseURL string
|
||||
realm string
|
||||
token string
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
func newPolarisHTTPClient(baseURL, realm, token string) *polarisHTTPClient {
|
||||
return &polarisHTTPClient{
|
||||
baseURL: baseURL,
|
||||
realm: realm,
|
||||
token: token,
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *polarisHTTPClient) doJSON(ctx context.Context, method, path string, body interface{}, out interface{}) error {
|
||||
return c.doJSONWithHeaders(ctx, method, path, body, out, nil)
|
||||
}
|
||||
|
||||
func (c *polarisHTTPClient) doJSONWithHeaders(ctx context.Context, method, path string, body interface{}, out interface{}, headers map[string]string) error {
|
||||
var reader io.Reader
|
||||
if body != nil {
|
||||
encoded, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode request body: %w", err)
|
||||
}
|
||||
reader = bytes.NewReader(encoded)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if c.realm != "" {
|
||||
req.Header.Set("Polaris-Realm", c.realm)
|
||||
}
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
for key, value := range headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
bodyBytes, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("request failed with status %d and could not read response body: %w", resp.StatusCode, readErr)
|
||||
}
|
||||
return fmt.Errorf("request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
|
||||
}
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
|
||||
return fmt.Errorf("decode response: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials, pathStyle bool) (*s3.Client, error) {
|
||||
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
if service == s3.ServiceID {
|
||||
return aws.Endpoint{
|
||||
URL: endpoint,
|
||||
SigningRegion: region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
}
|
||||
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
|
||||
})
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(ctx,
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
|
||||
config.WithEndpointResolverWithOptions(resolver),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.UsePathStyle = pathStyle
|
||||
}), nil
|
||||
}
|
||||
@@ -0,0 +1,764 @@
|
||||
package polaris
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
type polarisSession struct {
|
||||
catalogName string
|
||||
bucketName string
|
||||
token string
|
||||
baseLocation string
|
||||
}
|
||||
|
||||
type polarisTableSetup struct {
|
||||
namespace string
|
||||
table string
|
||||
dataKeyPrefix string
|
||||
s3Client *s3.Client
|
||||
}
|
||||
|
||||
type polarisCatalogClient struct {
|
||||
http *polarisHTTPClient
|
||||
catalog string
|
||||
}
|
||||
|
||||
type polarisCredentials struct {
|
||||
ClientID string `json:"clientId"`
|
||||
ClientSecret string `json:"clientSecret"`
|
||||
}
|
||||
|
||||
type createPrincipalResponse struct {
|
||||
Credentials polarisCredentials `json:"credentials"`
|
||||
}
|
||||
|
||||
type createCatalogRequest struct {
|
||||
Catalog polarisCatalog `json:"catalog"`
|
||||
}
|
||||
|
||||
type polarisCatalog struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
ReadOnly bool `json:"readOnly"`
|
||||
Properties map[string]string `json:"properties"`
|
||||
StorageConfigInfo polarisStorageConfig `json:"storageConfigInfo"`
|
||||
}
|
||||
|
||||
type polarisStorageConfig struct {
|
||||
StorageType string `json:"storageType"`
|
||||
AllowedLocations []string `json:"allowedLocations"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
EndpointInternal string `json:"endpointInternal"`
|
||||
StsEndpoint string `json:"stsEndpoint"`
|
||||
PathStyleAccess bool `json:"pathStyleAccess"`
|
||||
RoleArn string `json:"roleArn"`
|
||||
Region string `json:"region"`
|
||||
}
|
||||
|
||||
type createNamespaceRequest struct {
|
||||
Namespace []string `json:"namespace"`
|
||||
}
|
||||
|
||||
type createTableRequest struct {
|
||||
Name string `json:"name"`
|
||||
Location string `json:"location"`
|
||||
Schema icebergSchema `json:"schema"`
|
||||
PartitionSpec icebergPartition `json:"partition-spec"`
|
||||
SortOrder icebergSortOrder `json:"sort-order"`
|
||||
Properties map[string]string `json:"properties"`
|
||||
}
|
||||
|
||||
type icebergSchema struct {
|
||||
Type string `json:"type"`
|
||||
SchemaID int `json:"schema-id"`
|
||||
Fields []icebergSchemaField `json:"fields"`
|
||||
}
|
||||
|
||||
type icebergSchemaField struct {
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Required bool `json:"required"`
|
||||
}
|
||||
|
||||
type icebergPartition struct {
|
||||
SpecID int `json:"spec-id"`
|
||||
Fields []icebergPartitionField `json:"fields"`
|
||||
}
|
||||
|
||||
type icebergPartitionField struct {
|
||||
SourceID int `json:"source-id"`
|
||||
FieldID int `json:"field-id"`
|
||||
Name string `json:"name"`
|
||||
Transform string `json:"transform"`
|
||||
}
|
||||
|
||||
type icebergSortOrder struct {
|
||||
OrderID int `json:"order-id"`
|
||||
Fields []icebergSortField `json:"fields"`
|
||||
}
|
||||
|
||||
type icebergSortField struct {
|
||||
SourceID int `json:"source-id"`
|
||||
Direction string `json:"direction"`
|
||||
NullOrder string `json:"null-order"`
|
||||
}
|
||||
|
||||
type loadTableResponse struct {
|
||||
Config map[string]string `json:"config"`
|
||||
StorageCredentials []storageCredential `json:"storage-credentials"`
|
||||
}
|
||||
|
||||
type loadCredentialsResponse struct {
|
||||
StorageCredentials []storageCredential `json:"storage-credentials"`
|
||||
}
|
||||
|
||||
type storageCredential struct {
|
||||
Prefix string `json:"prefix"`
|
||||
Config map[string]string `json:"config"`
|
||||
}
|
||||
|
||||
func bootstrapPolarisTest(t *testing.T, env *TestEnvironment) (context.Context, context.CancelFunc, polarisSession, *polarisTableSetup, func()) {
|
||||
t.Helper()
|
||||
|
||||
t.Logf(">>> Starting SeaweedFS with Polaris configuration...")
|
||||
env.StartSeaweedFS(t)
|
||||
t.Logf(">>> SeaweedFS started.")
|
||||
|
||||
t.Logf(">>> Starting Polaris...")
|
||||
env.StartPolaris(t)
|
||||
t.Logf(">>> Polaris started.")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
session := newPolarisSession(t, ctx, env)
|
||||
setup, cleanup := setupPolarisTable(t, ctx, env, session)
|
||||
|
||||
return ctx, cancel, session, setup, cleanup
|
||||
}
|
||||
|
||||
func TestPolarisIntegration(t *testing.T) {
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
|
||||
defer cancel()
|
||||
defer cleanup()
|
||||
|
||||
objectKey := fmt.Sprintf("%s/hello-%d.txt", setup.dataKeyPrefix, time.Now().UnixNano())
|
||||
payload := []byte("polaris")
|
||||
|
||||
if _, err := setup.s3Client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(payload),
|
||||
}); err != nil {
|
||||
t.Fatalf("PutObject failed: %v", err)
|
||||
}
|
||||
|
||||
listObjects, err := setup.s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ListObjectsV2 failed: %v", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, obj := range listObjects.Contents {
|
||||
if aws.ToString(obj.Key) == objectKey {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("Object %s not found in list", objectKey)
|
||||
}
|
||||
|
||||
getResp, err := setup.s3Client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("GetObject failed: %v", err)
|
||||
}
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
_ = getResp.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("Read object body failed: %v", err)
|
||||
}
|
||||
if !bytes.Equal(body, payload) {
|
||||
t.Fatalf("Unexpected object payload: got %q want %q", string(body), string(payload))
|
||||
}
|
||||
|
||||
if _, err := setup.s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
}); err != nil {
|
||||
t.Fatalf("DeleteObject failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPolarisTableIntegration(t *testing.T) {
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
|
||||
defer cancel()
|
||||
defer cleanup()
|
||||
|
||||
objectKey := fmt.Sprintf("%s/part-%d.parquet", setup.dataKeyPrefix, time.Now().UnixNano())
|
||||
createResp, err := setup.s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CreateMultipartUpload failed: %v", err)
|
||||
}
|
||||
|
||||
uploadID := aws.ToString(createResp.UploadId)
|
||||
multipartCompleted := false
|
||||
defer func() {
|
||||
if uploadID == "" || multipartCompleted {
|
||||
return
|
||||
}
|
||||
_, _ = setup.s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
})
|
||||
}()
|
||||
|
||||
partSize := 5 * 1024 * 1024
|
||||
part1 := bytes.Repeat([]byte("a"), partSize)
|
||||
part2 := bytes.Repeat([]byte("b"), 1024*1024)
|
||||
|
||||
part1Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader(part1),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UploadPart 1 failed: %v", err)
|
||||
}
|
||||
|
||||
part2Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
PartNumber: aws.Int32(2),
|
||||
Body: bytes.NewReader(part2),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("UploadPart 2 failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = setup.s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadID),
|
||||
MultipartUpload: &s3types.CompletedMultipartUpload{
|
||||
Parts: []s3types.CompletedPart{
|
||||
{
|
||||
ETag: part1Resp.ETag,
|
||||
PartNumber: aws.Int32(1),
|
||||
},
|
||||
{
|
||||
ETag: part2Resp.ETag,
|
||||
PartNumber: aws.Int32(2),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteMultipartUpload failed: %v", err)
|
||||
}
|
||||
multipartCompleted = true
|
||||
|
||||
headResp, err := setup.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(session.bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("HeadObject after multipart upload failed: %v", err)
|
||||
}
|
||||
expectedSize := int64(len(part1) + len(part2))
|
||||
if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
|
||||
t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
|
||||
}
|
||||
}
|
||||
|
||||
func newPolarisSession(t *testing.T, ctx context.Context, env *TestEnvironment) polarisSession {
|
||||
t.Helper()
|
||||
|
||||
adminCreds := aws.Credentials{
|
||||
AccessKeyID: env.accessKey,
|
||||
SecretAccessKey: env.secretKey,
|
||||
Source: "polaris-admin",
|
||||
}
|
||||
adminS3, err := newS3Client(ctx, env.s3Endpoint(), polarisRegion, adminCreds, true)
|
||||
if err != nil {
|
||||
t.Fatalf("Create admin S3 client failed: %v", err)
|
||||
}
|
||||
|
||||
bucketName := fmt.Sprintf("polaris-bucket-%d", time.Now().UnixNano())
|
||||
if _, err := adminS3.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateBucket failed: %v", err)
|
||||
}
|
||||
policyDoc := map[string]interface{}{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": []map[string]interface{}{
|
||||
{
|
||||
"Sid": "AllowPolarisVendedAccess",
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:*",
|
||||
"Resource": []string{
|
||||
fmt.Sprintf("arn:aws:s3:::%s", bucketName),
|
||||
fmt.Sprintf("arn:aws:s3:::%s/polaris/*", bucketName),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
policyBytes, err := json.Marshal(policyDoc)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to marshal bucket policy: %v", err)
|
||||
}
|
||||
if _, err := adminS3.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Policy: aws.String(string(policyBytes)),
|
||||
}); err != nil {
|
||||
t.Fatalf("PutBucketPolicy failed: %v", err)
|
||||
}
|
||||
|
||||
rootToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), polarisRootClientID, polarisRootClientSecret)
|
||||
if err != nil {
|
||||
t.Fatalf("Polaris root token request failed: %v", err)
|
||||
}
|
||||
|
||||
managementClient := newPolarisHTTPClient(env.polarisEndpoint(), polarisRealm, rootToken)
|
||||
catalogName := fmt.Sprintf("polaris_catalog_%d", time.Now().UnixNano())
|
||||
baseLocation := fmt.Sprintf("s3://%s/polaris", bucketName)
|
||||
|
||||
catalogRequest := createCatalogRequest{
|
||||
Catalog: polarisCatalog{
|
||||
Name: catalogName,
|
||||
Type: "INTERNAL",
|
||||
ReadOnly: false,
|
||||
Properties: map[string]string{
|
||||
"default-base-location": baseLocation,
|
||||
},
|
||||
StorageConfigInfo: polarisStorageConfig{
|
||||
StorageType: "S3",
|
||||
AllowedLocations: []string{baseLocation},
|
||||
Endpoint: env.s3Endpoint(),
|
||||
EndpointInternal: env.s3InternalEndpoint(),
|
||||
StsEndpoint: env.s3InternalEndpoint(),
|
||||
PathStyleAccess: true,
|
||||
RoleArn: polarisRoleArn,
|
||||
Region: polarisRegion,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/catalogs", catalogRequest, nil); err != nil {
|
||||
t.Fatalf("Create catalog failed: %v", err)
|
||||
}
|
||||
|
||||
principalName := fmt.Sprintf("polaris_user_%d", time.Now().UnixNano())
|
||||
principalRoleName := fmt.Sprintf("polaris_principal_role_%d", time.Now().UnixNano())
|
||||
catalogRoleName := fmt.Sprintf("polaris_catalog_role_%d", time.Now().UnixNano())
|
||||
|
||||
var principalResp createPrincipalResponse
|
||||
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principals", map[string]interface{}{
|
||||
"principal": map[string]interface{}{
|
||||
"name": principalName,
|
||||
"properties": map[string]string{},
|
||||
},
|
||||
}, &principalResp); err != nil {
|
||||
t.Fatalf("Create principal failed: %v", err)
|
||||
}
|
||||
if principalResp.Credentials.ClientID == "" || principalResp.Credentials.ClientSecret == "" {
|
||||
t.Fatalf("Missing principal credentials in response")
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principal-roles", map[string]interface{}{
|
||||
"principalRole": map[string]interface{}{
|
||||
"name": principalRoleName,
|
||||
"properties": map[string]string{},
|
||||
},
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("Create principal role failed: %v", err)
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPost, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles", url.PathEscape(catalogName)), map[string]interface{}{
|
||||
"catalogRole": map[string]interface{}{
|
||||
"name": catalogRoleName,
|
||||
"properties": map[string]string{},
|
||||
},
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("Create catalog role failed: %v", err)
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principals/%s/principal-roles", url.PathEscape(principalName)), map[string]interface{}{
|
||||
"principalRole": map[string]interface{}{
|
||||
"name": principalRoleName,
|
||||
},
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("Assign principal role failed: %v", err)
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principal-roles/%s/catalog-roles/%s", url.PathEscape(principalRoleName), url.PathEscape(catalogName)), map[string]interface{}{
|
||||
"catalogRole": map[string]interface{}{
|
||||
"name": catalogRoleName,
|
||||
},
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("Assign catalog role failed: %v", err)
|
||||
}
|
||||
|
||||
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles/%s/grants", url.PathEscape(catalogName), url.PathEscape(catalogRoleName)), map[string]interface{}{
|
||||
"type": "catalog",
|
||||
"privilege": "CATALOG_MANAGE_CONTENT",
|
||||
}, nil); err != nil {
|
||||
t.Fatalf("Grant catalog privilege failed: %v", err)
|
||||
}
|
||||
|
||||
userToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), principalResp.Credentials.ClientID, principalResp.Credentials.ClientSecret)
|
||||
if err != nil {
|
||||
t.Fatalf("Polaris user token request failed: %v", err)
|
||||
}
|
||||
|
||||
return polarisSession{
|
||||
catalogName: catalogName,
|
||||
bucketName: bucketName,
|
||||
token: userToken,
|
||||
baseLocation: baseLocation,
|
||||
}
|
||||
}
|
||||
|
||||
func setupPolarisTable(t *testing.T, ctx context.Context, env *TestEnvironment, session polarisSession) (*polarisTableSetup, func()) {
|
||||
t.Helper()
|
||||
|
||||
catalogClient := newPolarisCatalogClient(env.polarisEndpoint(), polarisRealm, session.token, session.catalogName)
|
||||
namespace := fmt.Sprintf("polaris_ns_%d", time.Now().UnixNano())
|
||||
table := fmt.Sprintf("polaris_table_%d", time.Now().UnixNano())
|
||||
|
||||
if err := catalogClient.CreateNamespace(ctx, namespace); err != nil {
|
||||
t.Fatalf("CreateNamespace failed: %v", err)
|
||||
}
|
||||
|
||||
location := fmt.Sprintf("%s/%s/%s", session.baseLocation, namespace, table)
|
||||
if err := catalogClient.CreateTable(ctx, namespace, table, location); err != nil {
|
||||
t.Fatalf("CreateTable failed: %v", err)
|
||||
}
|
||||
|
||||
loadResp, err := catalogClient.LoadTable(ctx, namespace, table)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadTable failed: %v", err)
|
||||
}
|
||||
|
||||
credsResp, err := catalogClient.LoadCredentials(ctx, namespace, table)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadCredentials failed: %v", err)
|
||||
}
|
||||
if len(credsResp.StorageCredentials) == 0 {
|
||||
t.Fatalf("LoadCredentials returned no storage credentials")
|
||||
}
|
||||
|
||||
credentialSource := &loadTableResponse{
|
||||
Config: loadResp.Config,
|
||||
StorageCredentials: credsResp.StorageCredentials,
|
||||
}
|
||||
|
||||
dataPrefix := location + "/data"
|
||||
creds, endpoint, region, pathStyle, err := extractS3Credentials(credentialSource, dataPrefix, env.s3Endpoint(), polarisRegion)
|
||||
if err != nil {
|
||||
t.Fatalf("Extract vended credentials failed: %v", err)
|
||||
}
|
||||
dataKeyPrefix, err := s3URIToKeyPrefix(dataPrefix, session.bucketName)
|
||||
if err != nil {
|
||||
t.Fatalf("Invalid data prefix %s: %v", dataPrefix, err)
|
||||
}
|
||||
|
||||
s3Client, err := newS3Client(ctx, endpoint, region, creds, pathStyle)
|
||||
if err != nil {
|
||||
t.Fatalf("Create vended S3 client failed: %v", err)
|
||||
}
|
||||
|
||||
cleanup := func() {
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cleanupCancel()
|
||||
if err := catalogClient.DeleteTable(cleanupCtx, namespace, table); err != nil {
|
||||
t.Logf("DeleteTable failed: %v", err)
|
||||
}
|
||||
if err := catalogClient.DeleteNamespace(cleanupCtx, namespace); err != nil {
|
||||
t.Logf("DeleteNamespace failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &polarisTableSetup{
|
||||
namespace: namespace,
|
||||
table: table,
|
||||
dataKeyPrefix: dataKeyPrefix,
|
||||
s3Client: s3Client,
|
||||
}, cleanup
|
||||
}
|
||||
|
||||
func fetchPolarisToken(ctx context.Context, baseURL, clientID, clientSecret string) (string, error) {
|
||||
form := url.Values{}
|
||||
form.Set("grant_type", "client_credentials")
|
||||
form.Set("client_id", clientID)
|
||||
form.Set("client_secret", clientSecret)
|
||||
form.Set("scope", "PRINCIPAL_ROLE:ALL")
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/api/catalog/v1/oauth/tokens", strings.NewReader(form.Encode()))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create token request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("token request failed: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
return "", fmt.Errorf("token request failed with status %d and reading body: %w", resp.StatusCode, readErr)
|
||||
}
|
||||
return "", fmt.Errorf("token request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
|
||||
var tokenResp struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
|
||||
return "", fmt.Errorf("decode token response: %w", err)
|
||||
}
|
||||
if tokenResp.AccessToken == "" {
|
||||
return "", fmt.Errorf("missing access token in response")
|
||||
}
|
||||
|
||||
return tokenResp.AccessToken, nil
|
||||
}
|
||||
|
||||
func newPolarisCatalogClient(baseURL, realm, token, catalog string) *polarisCatalogClient {
|
||||
return &polarisCatalogClient{
|
||||
http: newPolarisHTTPClient(baseURL, realm, token),
|
||||
catalog: catalog,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) CreateNamespace(ctx context.Context, namespace string) error {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces", url.PathEscape(c.catalog))
|
||||
req := createNamespaceRequest{Namespace: []string{namespace}}
|
||||
return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) DeleteNamespace(ctx context.Context, namespace string) error {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s", url.PathEscape(c.catalog), url.PathEscape(namespace))
|
||||
return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) CreateTable(ctx context.Context, namespace, table, location string) error {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables", url.PathEscape(c.catalog), url.PathEscape(namespace))
|
||||
|
||||
req := createTableRequest{
|
||||
Name: table,
|
||||
Location: location,
|
||||
Schema: icebergSchema{
|
||||
Type: "struct",
|
||||
SchemaID: 0,
|
||||
Fields: []icebergSchemaField{
|
||||
{
|
||||
ID: 1,
|
||||
Name: "id",
|
||||
Type: "long",
|
||||
Required: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
PartitionSpec: icebergPartition{
|
||||
SpecID: 0,
|
||||
Fields: []icebergPartitionField{},
|
||||
},
|
||||
SortOrder: icebergSortOrder{
|
||||
OrderID: 0,
|
||||
Fields: []icebergSortField{},
|
||||
},
|
||||
Properties: map[string]string{
|
||||
"format-version": "2",
|
||||
},
|
||||
}
|
||||
|
||||
return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) DeleteTable(ctx context.Context, namespace, table string) error {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
|
||||
return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) LoadTable(ctx context.Context, namespace, table string) (*loadTableResponse, error) {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
|
||||
var resp loadTableResponse
|
||||
headers := map[string]string{
|
||||
"X-Iceberg-Access-Delegation": "vended-credentials",
|
||||
}
|
||||
if err := c.http.doJSONWithHeaders(ctx, http.MethodGet, path, nil, &resp, headers); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
func (c *polarisCatalogClient) LoadCredentials(ctx context.Context, namespace, table string) (*loadCredentialsResponse, error) {
|
||||
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s/credentials", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
|
||||
var resp loadCredentialsResponse
|
||||
if err := c.http.doJSON(ctx, http.MethodGet, path, nil, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
|
||||
func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoint, fallbackRegion string) (aws.Credentials, string, string, bool, error) {
|
||||
credentialConfig, err := selectStorageConfig(load, targetPrefix)
|
||||
if err != nil {
|
||||
return aws.Credentials{}, "", "", false, err
|
||||
}
|
||||
|
||||
lookupConfig := func(key string) string {
|
||||
if load != nil && load.Config != nil {
|
||||
if val, ok := load.Config[key]; ok && strings.TrimSpace(val) != "" {
|
||||
return strings.TrimSpace(val)
|
||||
}
|
||||
}
|
||||
if val, ok := credentialConfig[key]; ok && strings.TrimSpace(val) != "" {
|
||||
return strings.TrimSpace(val)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
accessKey := strings.TrimSpace(credentialConfig["s3.access-key-id"])
|
||||
secretKey := strings.TrimSpace(credentialConfig["s3.secret-access-key"])
|
||||
sessionToken := strings.TrimSpace(credentialConfig["s3.session-token"])
|
||||
if accessKey == "" || secretKey == "" {
|
||||
return aws.Credentials{}, "", "", false, fmt.Errorf("missing s3.access-key-id or s3.secret-access-key in selected storage credential")
|
||||
}
|
||||
|
||||
endpoint := lookupConfig("s3.endpoint")
|
||||
if endpoint == "" {
|
||||
endpoint = fallbackEndpoint
|
||||
}
|
||||
if endpoint != "" && !strings.HasPrefix(endpoint, "http") {
|
||||
endpoint = "http://" + endpoint
|
||||
}
|
||||
|
||||
region := lookupConfig("client.region")
|
||||
if region == "" {
|
||||
region = fallbackRegion
|
||||
}
|
||||
|
||||
pathStyle := true
|
||||
if value := lookupConfig("s3.path-style-access"); value != "" {
|
||||
pathStyle = strings.EqualFold(value, "true")
|
||||
}
|
||||
|
||||
return aws.Credentials{
|
||||
AccessKeyID: accessKey,
|
||||
SecretAccessKey: secretKey,
|
||||
SessionToken: sessionToken,
|
||||
Source: "polaris-vended",
|
||||
}, endpoint, region, pathStyle, nil
|
||||
}
|
||||
|
||||
func selectStorageConfig(load *loadTableResponse, targetPrefix string) (map[string]string, error) {
|
||||
if load == nil {
|
||||
return nil, fmt.Errorf("load table response is nil")
|
||||
}
|
||||
|
||||
switch len(load.StorageCredentials) {
|
||||
case 0:
|
||||
if load.Config == nil {
|
||||
return nil, fmt.Errorf("polaris returned no storage credentials or config")
|
||||
}
|
||||
return load.Config, nil
|
||||
case 1:
|
||||
cred := load.StorageCredentials[0]
|
||||
if cred.Config == nil {
|
||||
return nil, fmt.Errorf("storage credential for prefix %s returned nil config", cred.Prefix)
|
||||
}
|
||||
return cred.Config, nil
|
||||
default:
|
||||
if targetPrefix == "" {
|
||||
return nil, fmt.Errorf("multiple storage credentials (%d) returned but no target prefix provided", len(load.StorageCredentials))
|
||||
}
|
||||
normalizedTarget := normalizePrefix(targetPrefix)
|
||||
if normalizedTarget == "" {
|
||||
return nil, fmt.Errorf("target prefix %q normalized to empty string", targetPrefix)
|
||||
}
|
||||
var bestConfig map[string]string
|
||||
bestLen := -1
|
||||
for _, cred := range load.StorageCredentials {
|
||||
if cred.Config == nil {
|
||||
continue
|
||||
}
|
||||
prefix := normalizePrefix(cred.Prefix)
|
||||
if prefix == "" {
|
||||
if bestLen < 0 {
|
||||
bestLen = 0
|
||||
bestConfig = cred.Config
|
||||
}
|
||||
continue
|
||||
}
|
||||
if normalizedTarget == prefix || strings.HasPrefix(normalizedTarget, prefix+"/") {
|
||||
if len(prefix) > bestLen {
|
||||
bestLen = len(prefix)
|
||||
bestConfig = cred.Config
|
||||
}
|
||||
}
|
||||
}
|
||||
if bestConfig != nil {
|
||||
return bestConfig, nil
|
||||
}
|
||||
return nil, fmt.Errorf("none of the %d storage credentials matched prefix %s", len(load.StorageCredentials), targetPrefix)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePrefix(prefix string) string {
|
||||
p := strings.TrimSpace(prefix)
|
||||
p = strings.TrimSuffix(p, "/")
|
||||
return p
|
||||
}
|
||||
|
||||
func s3URIToKeyPrefix(uri, bucket string) (string, error) {
|
||||
prefix := "s3://" + bucket + "/"
|
||||
if !strings.HasPrefix(uri, prefix) {
|
||||
return "", fmt.Errorf("uri %q does not match bucket %q", uri, bucket)
|
||||
}
|
||||
keyPrefix := strings.TrimPrefix(uri, prefix)
|
||||
keyPrefix = strings.TrimPrefix(keyPrefix, "/")
|
||||
if keyPrefix == "" {
|
||||
return "", fmt.Errorf("empty key prefix in uri %q", uri)
|
||||
}
|
||||
return keyPrefix, nil
|
||||
}
|
||||
@@ -23,8 +23,6 @@ func TestAdminStatusAndHealthz(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("create status request: %v", err)
|
||||
}
|
||||
statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1")
|
||||
|
||||
statusResp := framework.DoRequest(t, client, statusReq)
|
||||
statusBody := framework.ReadAllAndClose(t, statusResp)
|
||||
|
||||
@@ -34,8 +32,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
|
||||
if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
|
||||
t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got)
|
||||
}
|
||||
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" {
|
||||
t.Fatalf("expected echoed request id, got %q", got)
|
||||
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
|
||||
t.Fatal("expected server-generated request id in response header")
|
||||
}
|
||||
|
||||
var payload map[string]interface{}
|
||||
@@ -49,7 +47,6 @@ func TestAdminStatusAndHealthz(t *testing.T) {
|
||||
}
|
||||
|
||||
healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz")
|
||||
healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2")
|
||||
healthResp := framework.DoRequest(t, client, healthReq)
|
||||
_ = framework.ReadAllAndClose(t, healthResp)
|
||||
if healthResp.StatusCode != http.StatusOK {
|
||||
@@ -58,8 +55,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
|
||||
if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
|
||||
t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got)
|
||||
}
|
||||
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" {
|
||||
t.Fatalf("expected /healthz echoed request id, got %q", got)
|
||||
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
|
||||
t.Fatal("expected /healthz server-generated request id in response header")
|
||||
}
|
||||
|
||||
uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
|
||||
|
||||
@@ -80,6 +80,11 @@ type AccessKeyInfo struct {
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type CreateAccessKeyRequest struct {
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
}
|
||||
|
||||
type UpdateAccessKeyStatusRequest struct {
|
||||
Status string `json:"status" binding:"required"`
|
||||
}
|
||||
@@ -90,6 +95,7 @@ type UserDetails struct {
|
||||
Actions []string `json:"actions"`
|
||||
PolicyNames []string `json:"policy_names"`
|
||||
AccessKeys []AccessKeyInfo `json:"access_keys"`
|
||||
Groups []string `json:"groups"`
|
||||
}
|
||||
|
||||
type FilerNode struct {
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
|
||||
)
|
||||
|
||||
const (
|
||||
adminLockName = cluster.AdminShellLockName
|
||||
adminLockClientName = "admin-plugin"
|
||||
)
|
||||
|
||||
// AdminLockManager coordinates exclusive admin locks with reference counting.
|
||||
// It is safe for concurrent use.
|
||||
type AdminLockManager struct {
|
||||
locker *exclusive_locks.ExclusiveLocker
|
||||
clientName string
|
||||
|
||||
mu sync.Mutex
|
||||
cond *sync.Cond
|
||||
acquiring bool
|
||||
holdCount int
|
||||
|
||||
lastAcquiredAt time.Time
|
||||
lastReleasedAt time.Time
|
||||
waitingSince time.Time
|
||||
waitingReason string
|
||||
currentReason string
|
||||
}
|
||||
|
||||
func NewAdminLockManager(masterClient *wdclient.MasterClient, clientName string) *AdminLockManager {
|
||||
if masterClient == nil {
|
||||
return nil
|
||||
}
|
||||
if clientName == "" {
|
||||
clientName = adminLockClientName
|
||||
}
|
||||
manager := &AdminLockManager{
|
||||
locker: exclusive_locks.NewExclusiveLocker(masterClient, adminLockName),
|
||||
clientName: clientName,
|
||||
}
|
||||
manager.cond = sync.NewCond(&manager.mu)
|
||||
return manager
|
||||
}
|
||||
|
||||
func (m *AdminLockManager) Acquire(reason string) (func(), error) {
|
||||
if m == nil || m.locker == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
if reason != "" {
|
||||
m.locker.SetMessage(reason)
|
||||
m.currentReason = reason
|
||||
}
|
||||
for m.acquiring {
|
||||
m.cond.Wait()
|
||||
}
|
||||
if m.holdCount == 0 {
|
||||
m.acquiring = true
|
||||
m.waitingSince = time.Now().UTC()
|
||||
m.waitingReason = reason
|
||||
m.mu.Unlock()
|
||||
m.locker.RequestLock(m.clientName)
|
||||
m.mu.Lock()
|
||||
m.acquiring = false
|
||||
m.holdCount = 1
|
||||
m.lastAcquiredAt = time.Now().UTC()
|
||||
m.waitingSince = time.Time{}
|
||||
m.waitingReason = ""
|
||||
m.cond.Broadcast()
|
||||
m.mu.Unlock()
|
||||
return m.Release, nil
|
||||
}
|
||||
m.holdCount++
|
||||
if reason != "" {
|
||||
m.currentReason = reason
|
||||
}
|
||||
m.mu.Unlock()
|
||||
return m.Release, nil
|
||||
}
|
||||
|
||||
func (m *AdminLockManager) Release() {
|
||||
if m == nil || m.locker == nil {
|
||||
return
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
if m.holdCount <= 0 {
|
||||
m.mu.Unlock()
|
||||
return
|
||||
}
|
||||
m.holdCount--
|
||||
shouldRelease := m.holdCount == 0
|
||||
m.mu.Unlock()
|
||||
|
||||
if shouldRelease {
|
||||
m.mu.Lock()
|
||||
m.lastReleasedAt = time.Now().UTC()
|
||||
m.currentReason = ""
|
||||
m.mu.Unlock()
|
||||
m.locker.ReleaseLock()
|
||||
}
|
||||
}
|
||||
|
||||
type LockStatus struct {
|
||||
Held bool `json:"held"`
|
||||
HoldCount int `json:"hold_count"`
|
||||
Acquiring bool `json:"acquiring"`
|
||||
Message string `json:"message,omitempty"`
|
||||
WaitingReason string `json:"waiting_reason,omitempty"`
|
||||
LastAcquiredAt *time.Time `json:"last_acquired_at,omitempty"`
|
||||
LastReleasedAt *time.Time `json:"last_released_at,omitempty"`
|
||||
WaitingSince *time.Time `json:"waiting_since,omitempty"`
|
||||
}
|
||||
|
||||
func (m *AdminLockManager) Status() LockStatus {
|
||||
if m == nil {
|
||||
return LockStatus{}
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
status := LockStatus{
|
||||
Held: m.holdCount > 0,
|
||||
HoldCount: m.holdCount,
|
||||
Acquiring: m.acquiring,
|
||||
Message: m.currentReason,
|
||||
WaitingReason: m.waitingReason,
|
||||
}
|
||||
if !m.lastAcquiredAt.IsZero() {
|
||||
at := m.lastAcquiredAt
|
||||
status.LastAcquiredAt = &at
|
||||
}
|
||||
if !m.lastReleasedAt.IsZero() {
|
||||
at := m.lastReleasedAt
|
||||
status.LastReleasedAt = &at
|
||||
}
|
||||
if !m.waitingSince.IsZero() {
|
||||
at := m.waitingSince
|
||||
status.WaitingSince = &at
|
||||
}
|
||||
return status
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
|
||||
)
|
||||
|
||||
const adminPresenceClientName = "admin-server"
|
||||
|
||||
type adminPresenceLock struct {
|
||||
locker *exclusive_locks.ExclusiveLocker
|
||||
stopCh chan struct{}
|
||||
}
|
||||
|
||||
func newAdminPresenceLock(masterClient *wdclient.MasterClient) *adminPresenceLock {
|
||||
if masterClient == nil {
|
||||
return nil
|
||||
}
|
||||
return &adminPresenceLock{
|
||||
locker: exclusive_locks.NewExclusiveLocker(masterClient, cluster.AdminServerPresenceLockName),
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (l *adminPresenceLock) Start() {
|
||||
if l == nil || l.locker == nil {
|
||||
return
|
||||
}
|
||||
l.locker.SetMessage("admin server connected")
|
||||
go func() {
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if !l.locker.IsLocked() {
|
||||
l.locker.RequestLock(adminPresenceClientName)
|
||||
}
|
||||
select {
|
||||
case <-l.stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (l *adminPresenceLock) Stop() {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-l.stopCh:
|
||||
default:
|
||||
close(l.stopCh)
|
||||
}
|
||||
if l.locker != nil {
|
||||
l.locker.ReleaseLock()
|
||||
}
|
||||
}
|
||||
+299
-30
@@ -2,7 +2,9 @@ package dash
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -11,6 +13,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
|
||||
adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
clustermaintenance "github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
@@ -98,6 +101,9 @@ type AdminServer struct {
|
||||
// Maintenance system
|
||||
maintenanceManager *maintenance.MaintenanceManager
|
||||
plugin *adminplugin.Plugin
|
||||
pluginLock *AdminLockManager
|
||||
adminPresenceLock *adminPresenceLock
|
||||
expireJobHandler func(jobID string, reason string) (*adminplugin.TrackedJob, bool, error)
|
||||
|
||||
// Topic retention purger
|
||||
topicRetentionPurger *TopicRetentionPurger
|
||||
@@ -134,6 +140,12 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
|
||||
ctx := context.Background()
|
||||
go masterClient.KeepConnectedToMaster(ctx)
|
||||
|
||||
lockManager := NewAdminLockManager(masterClient, adminLockClientName)
|
||||
presenceLock := newAdminPresenceLock(masterClient)
|
||||
if presenceLock != nil {
|
||||
presenceLock.Start()
|
||||
}
|
||||
|
||||
server := &AdminServer{
|
||||
masterClient: masterClient,
|
||||
templateFS: templateFS,
|
||||
@@ -145,6 +157,8 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
|
||||
collectionStatsCacheThreshold: defaultStatsCacheTimeout,
|
||||
s3TablesManager: newS3TablesManager(),
|
||||
icebergPort: icebergPort,
|
||||
pluginLock: lockManager,
|
||||
adminPresenceLock: presenceLock,
|
||||
}
|
||||
|
||||
// Initialize topic retention purger
|
||||
@@ -223,20 +237,19 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
|
||||
}()
|
||||
}
|
||||
|
||||
plugin, err := adminplugin.New(adminplugin.Options{
|
||||
pluginOpts := adminplugin.Options{
|
||||
DataDir: dataDir,
|
||||
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
|
||||
return server.buildDefaultPluginClusterContext(), nil
|
||||
},
|
||||
})
|
||||
LockManager: lockManager,
|
||||
ConfigDefaultsProvider: server.enrichConfigDefaults,
|
||||
}
|
||||
plugin, err := adminplugin.New(pluginOpts)
|
||||
if err != nil && dataDir != "" {
|
||||
glog.Warningf("Failed to initialize plugin with dataDir=%q: %v. Falling back to in-memory plugin state.", dataDir, err)
|
||||
plugin, err = adminplugin.New(adminplugin.Options{
|
||||
DataDir: "",
|
||||
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
|
||||
return server.buildDefaultPluginClusterContext(), nil
|
||||
},
|
||||
})
|
||||
pluginOpts.DataDir = ""
|
||||
plugin, err = adminplugin.New(pluginOpts)
|
||||
}
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to initialize plugin: %v", err)
|
||||
@@ -260,6 +273,89 @@ func (s *AdminServer) loadTaskConfigurationsFromPersistence() {
|
||||
configUpdateRegistry.UpdateAllConfigs(s.configPersistence)
|
||||
}
|
||||
|
||||
// enrichConfigDefaults is called by the plugin when bootstrapping a job type's
|
||||
// default config from its descriptor. For admin_script, it fetches maintenance
|
||||
// scripts from the master and uses them as the script default.
|
||||
//
|
||||
// MIGRATION: This exists to help users migrate from master.toml [master.maintenance]
|
||||
// to the admin script plugin worker. Remove after March 2027.
|
||||
func (s *AdminServer) enrichConfigDefaults(cfg *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig {
|
||||
if cfg.JobType != "admin_script" {
|
||||
return cfg
|
||||
}
|
||||
|
||||
var maintenanceScripts string
|
||||
var sleepMinutes uint32
|
||||
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
maintenanceScripts = resp.MaintenanceScripts
|
||||
sleepMinutes = resp.MaintenanceSleepMinutes
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("Could not fetch master configuration for admin_script defaults: %v", err)
|
||||
return cfg
|
||||
}
|
||||
|
||||
script := cleanMaintenanceScript(maintenanceScripts)
|
||||
if script == "" {
|
||||
return cfg
|
||||
}
|
||||
|
||||
interval := int64(sleepMinutes)
|
||||
if interval <= 0 {
|
||||
interval = clustermaintenance.DefaultMaintenanceSleepMinutes
|
||||
}
|
||||
|
||||
glog.V(0).Infof("Enriching admin_script defaults from master maintenance scripts (interval=%dm)", interval)
|
||||
|
||||
if cfg.AdminConfigValues == nil {
|
||||
cfg.AdminConfigValues = make(map[string]*plugin_pb.ConfigValue)
|
||||
}
|
||||
cfg.AdminConfigValues["script"] = &plugin_pb.ConfigValue{
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: script},
|
||||
}
|
||||
cfg.AdminConfigValues["run_interval_minutes"] = &plugin_pb.ConfigValue{
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: interval},
|
||||
}
|
||||
cfg.UpdatedBy = "master_migration"
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
// cleanMaintenanceScript strips lock/unlock commands and normalizes a
|
||||
// maintenance script string for use with the admin script plugin worker.
|
||||
//
|
||||
// MIGRATION: Used by enrichConfigDefaults. Remove after March 2027.
|
||||
func cleanMaintenanceScript(script string) string {
|
||||
script = strings.ReplaceAll(script, "\r\n", "\n")
|
||||
var lines []string
|
||||
for _, line := range strings.Split(script, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
|
||||
continue
|
||||
}
|
||||
// Strip inline comments (e.g., "lock # migration note")
|
||||
if idx := strings.Index(trimmed, "#"); idx >= 0 {
|
||||
trimmed = strings.TrimSpace(trimmed[:idx])
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
firstToken := strings.ToLower(strings.Fields(trimmed)[0])
|
||||
if firstToken == "lock" || firstToken == "unlock" {
|
||||
continue
|
||||
}
|
||||
lines = append(lines, trimmed)
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// GetCredentialManager returns the credential manager
|
||||
func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
|
||||
return s.credentialManager
|
||||
@@ -277,8 +373,21 @@ func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
|
||||
|
||||
// InvalidateCache method moved to cluster_topology.go
|
||||
|
||||
// GetS3BucketsData retrieves all Object Store buckets and aggregates total storage metrics
|
||||
func (s *AdminServer) GetS3BucketsData() (S3BucketsData, error) {
|
||||
// GetS3BucketsData retrieves Object Store buckets with pagination and sorting
|
||||
func (s *AdminServer) GetS3BucketsData(page, pageSize int, sortBy, sortOrder string) (S3BucketsData, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if pageSize < 1 || pageSize > 1000 {
|
||||
pageSize = 100
|
||||
}
|
||||
if sortBy == "" {
|
||||
sortBy = "name"
|
||||
}
|
||||
if sortOrder == "" {
|
||||
sortOrder = "asc"
|
||||
}
|
||||
|
||||
buckets, err := s.GetS3Buckets()
|
||||
if err != nil {
|
||||
return S3BucketsData{}, err
|
||||
@@ -289,14 +398,97 @@ func (s *AdminServer) GetS3BucketsData() (S3BucketsData, error) {
|
||||
totalSize += bucket.PhysicalSize
|
||||
}
|
||||
|
||||
totalBuckets := len(buckets)
|
||||
|
||||
// Sort buckets
|
||||
s.sortBuckets(buckets, sortBy, sortOrder)
|
||||
|
||||
// Calculate pagination
|
||||
totalPages := (totalBuckets + pageSize - 1) / pageSize
|
||||
if totalPages == 0 {
|
||||
totalPages = 1
|
||||
}
|
||||
if page > totalPages {
|
||||
page = totalPages
|
||||
}
|
||||
|
||||
startIndex := (page - 1) * pageSize
|
||||
endIndex := startIndex + pageSize
|
||||
if startIndex >= totalBuckets {
|
||||
buckets = []S3Bucket{}
|
||||
} else {
|
||||
if endIndex > totalBuckets {
|
||||
endIndex = totalBuckets
|
||||
}
|
||||
buckets = buckets[startIndex:endIndex]
|
||||
}
|
||||
|
||||
return S3BucketsData{
|
||||
Buckets: buckets,
|
||||
TotalBuckets: len(buckets),
|
||||
TotalBuckets: totalBuckets,
|
||||
TotalSize: totalSize,
|
||||
LastUpdated: time.Now(),
|
||||
CurrentPage: page,
|
||||
TotalPages: totalPages,
|
||||
PageSize: pageSize,
|
||||
SortBy: sortBy,
|
||||
SortOrder: sortOrder,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// sortBuckets sorts the bucket slice in place by the given field and order
|
||||
func (s *AdminServer) sortBuckets(buckets []S3Bucket, sortBy, sortOrder string) {
|
||||
desc := sortOrder == "desc"
|
||||
sort.Slice(buckets, func(i, j int) bool {
|
||||
a, b := buckets[i], buckets[j]
|
||||
switch sortBy {
|
||||
case "owner":
|
||||
if a.Owner != b.Owner {
|
||||
if desc {
|
||||
return a.Owner > b.Owner
|
||||
}
|
||||
return a.Owner < b.Owner
|
||||
}
|
||||
case "created":
|
||||
if !a.CreatedAt.Equal(b.CreatedAt) {
|
||||
if desc {
|
||||
return a.CreatedAt.After(b.CreatedAt)
|
||||
}
|
||||
return a.CreatedAt.Before(b.CreatedAt)
|
||||
}
|
||||
case "objects":
|
||||
if a.ObjectCount != b.ObjectCount {
|
||||
if desc {
|
||||
return a.ObjectCount > b.ObjectCount
|
||||
}
|
||||
return a.ObjectCount < b.ObjectCount
|
||||
}
|
||||
case "logical_size":
|
||||
if a.LogicalSize != b.LogicalSize {
|
||||
if desc {
|
||||
return a.LogicalSize > b.LogicalSize
|
||||
}
|
||||
return a.LogicalSize < b.LogicalSize
|
||||
}
|
||||
case "physical_size":
|
||||
if a.PhysicalSize != b.PhysicalSize {
|
||||
if desc {
|
||||
return a.PhysicalSize > b.PhysicalSize
|
||||
}
|
||||
return a.PhysicalSize < b.PhysicalSize
|
||||
}
|
||||
}
|
||||
// Tie-breaker: sort by name (also the default/primary for sortBy=="name")
|
||||
if a.Name != b.Name {
|
||||
if desc {
|
||||
return a.Name > b.Name
|
||||
}
|
||||
return a.Name < b.Name
|
||||
}
|
||||
return false
|
||||
})
|
||||
}
|
||||
|
||||
// GetS3Buckets retrieves all Object Store buckets from the filer and collects size/object data from collections
|
||||
func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
var buckets []S3Bucket
|
||||
@@ -312,28 +504,48 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
|
||||
// Now list buckets from the filer and match with collection data
|
||||
err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
// List buckets by looking at the buckets directory
|
||||
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
|
||||
Directory: filerConfig.BucketsPath,
|
||||
Prefix: "",
|
||||
StartFromFileName: "",
|
||||
InclusiveStartFrom: false,
|
||||
Limit: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Paginate through all buckets in the buckets directory
|
||||
const listPageSize = 1000
|
||||
startFrom := ""
|
||||
var snapshotTsNs int64
|
||||
for {
|
||||
resp, err := stream.Recv()
|
||||
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
|
||||
Directory: filerConfig.BucketsPath,
|
||||
Prefix: "",
|
||||
StartFromFileName: startFrom,
|
||||
InclusiveStartFrom: false,
|
||||
Limit: listPageSize,
|
||||
SnapshotTsNs: snapshotTsNs,
|
||||
})
|
||||
if err != nil {
|
||||
if err.Error() == "EOF" {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if resp.Entry.IsDirectory {
|
||||
pageCount := 0
|
||||
lastName := ""
|
||||
for {
|
||||
resp, err := stream.Recv()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if snapshotTsNs == 0 && resp.SnapshotTsNs != 0 {
|
||||
snapshotTsNs = resp.SnapshotTsNs
|
||||
}
|
||||
|
||||
if resp.Entry == nil {
|
||||
continue
|
||||
}
|
||||
lastName = resp.Entry.Name
|
||||
pageCount++
|
||||
|
||||
if !resp.Entry.IsDirectory {
|
||||
continue
|
||||
}
|
||||
|
||||
bucketName := resp.Entry.Name
|
||||
if strings.HasPrefix(bucketName, ".") {
|
||||
// Skip internal/system directories from Object Store bucket listing.
|
||||
@@ -386,13 +598,18 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
}
|
||||
}
|
||||
|
||||
var createdAt, lastModified time.Time
|
||||
if resp.Entry.Attributes != nil {
|
||||
createdAt = time.Unix(resp.Entry.Attributes.Crtime, 0)
|
||||
lastModified = time.Unix(resp.Entry.Attributes.Mtime, 0)
|
||||
}
|
||||
bucket := S3Bucket{
|
||||
Name: bucketName,
|
||||
CreatedAt: time.Unix(resp.Entry.Attributes.Crtime, 0),
|
||||
CreatedAt: createdAt,
|
||||
LogicalSize: logicalSize,
|
||||
PhysicalSize: physicalSize,
|
||||
ObjectCount: objectCount,
|
||||
LastModified: time.Unix(resp.Entry.Attributes.Mtime, 0),
|
||||
LastModified: lastModified,
|
||||
Quota: quota,
|
||||
QuotaEnabled: quotaEnabled,
|
||||
VersioningStatus: versioningStatus,
|
||||
@@ -403,6 +620,12 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
}
|
||||
buckets = append(buckets, bucket)
|
||||
}
|
||||
|
||||
// If we received fewer entries than the page size, we've listed everything
|
||||
if pageCount < listPageSize {
|
||||
break
|
||||
}
|
||||
startFrom = lastName
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -889,6 +1112,13 @@ func (s *AdminServer) GetPlugin() *adminplugin.Plugin {
|
||||
return s.plugin
|
||||
}
|
||||
|
||||
func (s *AdminServer) acquirePluginLock(reason string) (func(), error) {
|
||||
if s == nil || s.pluginLock == nil {
|
||||
return func() {}, nil
|
||||
}
|
||||
return s.pluginLock.Acquire(reason)
|
||||
}
|
||||
|
||||
// RequestPluginJobTypeDescriptor asks one worker for job type schema and returns the descriptor.
|
||||
func (s *AdminServer) RequestPluginJobTypeDescriptor(ctx context.Context, jobType string, forceRefresh bool) (*plugin_pb.JobTypeDescriptor, error) {
|
||||
if s.plugin == nil {
|
||||
@@ -931,6 +1161,13 @@ func (s *AdminServer) RunPluginDetection(
|
||||
if s.plugin == nil {
|
||||
return nil, fmt.Errorf("plugin is not enabled")
|
||||
}
|
||||
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if releaseLock != nil {
|
||||
defer releaseLock()
|
||||
}
|
||||
return s.plugin.RunDetection(ctx, jobType, clusterContext, maxResults)
|
||||
}
|
||||
|
||||
@@ -956,6 +1193,13 @@ func (s *AdminServer) RunPluginDetectionWithReport(
|
||||
if s.plugin == nil {
|
||||
return nil, fmt.Errorf("plugin is not enabled")
|
||||
}
|
||||
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if releaseLock != nil {
|
||||
defer releaseLock()
|
||||
}
|
||||
return s.plugin.RunDetectionWithReport(ctx, jobType, clusterContext, maxResults)
|
||||
}
|
||||
|
||||
@@ -969,6 +1213,17 @@ func (s *AdminServer) ExecutePluginJob(
|
||||
if s.plugin == nil {
|
||||
return nil, fmt.Errorf("plugin is not enabled")
|
||||
}
|
||||
jobType := ""
|
||||
if job != nil {
|
||||
jobType = strings.TrimSpace(job.JobType)
|
||||
}
|
||||
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin execution %s", jobType))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if releaseLock != nil {
|
||||
defer releaseLock()
|
||||
}
|
||||
return s.plugin.ExecuteJob(ctx, job, clusterContext, attempt)
|
||||
}
|
||||
|
||||
@@ -1020,6 +1275,17 @@ func (s *AdminServer) GetPluginJobDetail(jobID string, activityLimit, relatedLim
|
||||
return s.plugin.BuildJobDetail(jobID, activityLimit, relatedLimit)
|
||||
}
|
||||
|
||||
// ExpirePluginJob marks an active plugin job as failed so it no longer blocks scheduling.
|
||||
func (s *AdminServer) ExpirePluginJob(jobID, reason string) (*adminplugin.TrackedJob, bool, error) {
|
||||
if handler := s.expireJobHandler; handler != nil {
|
||||
return handler(jobID, reason)
|
||||
}
|
||||
if s.plugin == nil {
|
||||
return nil, false, fmt.Errorf("plugin is not enabled")
|
||||
}
|
||||
return s.plugin.ExpireJob(jobID, reason)
|
||||
}
|
||||
|
||||
// ListPluginActivities returns plugin job activities for monitoring.
|
||||
func (s *AdminServer) ListPluginActivities(jobType string, limit int) []adminplugin.JobActivity {
|
||||
if s.plugin == nil {
|
||||
@@ -1236,6 +1502,9 @@ func (s *AdminServer) Shutdown() {
|
||||
|
||||
// Stop maintenance manager
|
||||
s.StopMaintenanceManager()
|
||||
if s.adminPresenceLock != nil {
|
||||
s.adminPresenceLock.Stop()
|
||||
}
|
||||
|
||||
if s.plugin != nil {
|
||||
s.plugin.Shutdown()
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
// MIGRATION: Tests for enrichConfigDefaults helpers. Remove after March 2027.
|
||||
package dash
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCleanMaintenanceScript(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
input: "",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "only lock unlock",
|
||||
input: " lock\n unlock\n",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "strips lock and unlock",
|
||||
input: " lock\n ec.balance -apply\n volume.fix.replication -apply\n unlock\n",
|
||||
expected: "ec.balance -apply\nvolume.fix.replication -apply",
|
||||
},
|
||||
{
|
||||
name: "case insensitive lock",
|
||||
input: "Lock\nec.balance -apply\nUNLOCK",
|
||||
expected: "ec.balance -apply",
|
||||
},
|
||||
{
|
||||
name: "preserves comments removal",
|
||||
input: "lock\n# a comment\nec.balance -apply\nunlock",
|
||||
expected: "ec.balance -apply",
|
||||
},
|
||||
{
|
||||
name: "no lock unlock present",
|
||||
input: "ec.balance -apply\nvolume.fix.replication -apply",
|
||||
expected: "ec.balance -apply\nvolume.fix.replication -apply",
|
||||
},
|
||||
{
|
||||
name: "windows line endings",
|
||||
input: "lock\r\nec.balance -apply\r\nunlock\r\n",
|
||||
expected: "ec.balance -apply",
|
||||
},
|
||||
{
|
||||
name: "lock with inline comment",
|
||||
input: "lock # migration\nec.balance -apply\nunlock # done",
|
||||
expected: "ec.balance -apply",
|
||||
},
|
||||
{
|
||||
name: "command with inline comment preserved",
|
||||
input: "lock\nec.balance -apply # rebalance shards\nunlock",
|
||||
expected: "ec.balance -apply",
|
||||
},
|
||||
{
|
||||
name: "only inline comment after stripping",
|
||||
input: "# full line comment\n # indented comment\n",
|
||||
expected: "",
|
||||
},
|
||||
{
|
||||
name: "typical master default",
|
||||
input: "\n lock\n ec.encode -fullPercent=95 -quietFor=1h\n ec.rebuild -apply\n ec.balance -apply\n fs.log.purge -daysAgo=7\n volume.deleteEmpty -quietFor=24h -apply\n volume.balance -apply\n volume.fix.replication -apply\n s3.clean.uploads -timeAgo=24h\n unlock\n",
|
||||
expected: "ec.encode -fullPercent=95 -quietFor=1h\nec.rebuild -apply\nec.balance -apply\nfs.log.purge -daysAgo=7\nvolume.deleteEmpty -quietFor=24h -apply\nvolume.balance -apply\nvolume.fix.replication -apply\ns3.clean.uploads -timeAgo=24h",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := cleanMaintenanceScript(tt.input)
|
||||
if got != tt.expected {
|
||||
t.Errorf("cleanMaintenanceScript(%q) = %q, want %q", tt.input, got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,15 @@ type S3BucketsData struct {
|
||||
TotalBuckets int `json:"total_buckets"`
|
||||
TotalSize int64 `json:"total_size"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
|
||||
// Pagination
|
||||
CurrentPage int `json:"current_page"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
PageSize int `json:"page_size"`
|
||||
|
||||
// Sorting
|
||||
SortBy string `json:"sort_by"`
|
||||
SortOrder string `json:"sort_order"`
|
||||
}
|
||||
|
||||
type CreateBucketRequest struct {
|
||||
@@ -48,7 +57,7 @@ type CreateBucketRequest struct {
|
||||
func (s *AdminServer) ShowS3Buckets(w http.ResponseWriter, r *http.Request) {
|
||||
username := UsernameFromContext(r.Context())
|
||||
|
||||
data, err := s.GetS3BucketsData()
|
||||
data, err := s.GetS3BucketsData(1, 100, "name", "asc")
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get Object Store buckets: "+err.Error())
|
||||
return
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
)
|
||||
|
||||
// cloneGroup creates a deep copy of an iam_pb.Group to avoid mutating stored state.
|
||||
func cloneGroup(g *iam_pb.Group) *iam_pb.Group {
|
||||
clone := &iam_pb.Group{
|
||||
Name: g.Name,
|
||||
Disabled: g.Disabled,
|
||||
}
|
||||
if g.Members != nil {
|
||||
clone.Members = make([]string, len(g.Members))
|
||||
copy(clone.Members, g.Members)
|
||||
}
|
||||
if g.PolicyNames != nil {
|
||||
clone.PolicyNames = make([]string, len(g.PolicyNames))
|
||||
copy(clone.PolicyNames, g.PolicyNames)
|
||||
}
|
||||
return clone
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetGroups(ctx context.Context) ([]GroupData, error) {
|
||||
if s.credentialManager == nil {
|
||||
return nil, fmt.Errorf("credential manager not available")
|
||||
}
|
||||
|
||||
groupNames, err := s.credentialManager.ListGroups(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list groups: %w", err)
|
||||
}
|
||||
|
||||
var groups []GroupData
|
||||
for _, name := range groupNames {
|
||||
g, err := s.credentialManager.GetGroup(ctx, name)
|
||||
if err != nil {
|
||||
if errors.Is(err, credential.ErrGroupNotFound) {
|
||||
glog.V(1).Infof("Group %s listed but not found, skipping", name)
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("failed to get group %s: %w", name, err)
|
||||
}
|
||||
status := "enabled"
|
||||
if g.Disabled {
|
||||
status = "disabled"
|
||||
}
|
||||
groups = append(groups, GroupData{
|
||||
Name: g.Name,
|
||||
MemberCount: len(g.Members),
|
||||
PolicyCount: len(g.PolicyNames),
|
||||
Status: status,
|
||||
Members: g.Members,
|
||||
PolicyNames: g.PolicyNames,
|
||||
})
|
||||
}
|
||||
return groups, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetGroupDetails(ctx context.Context, name string) (*GroupData, error) {
|
||||
if s.credentialManager == nil {
|
||||
return nil, fmt.Errorf("credential manager not available")
|
||||
}
|
||||
|
||||
g, err := s.credentialManager.GetGroup(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get group: %w", err)
|
||||
}
|
||||
status := "enabled"
|
||||
if g.Disabled {
|
||||
status = "disabled"
|
||||
}
|
||||
return &GroupData{
|
||||
Name: g.Name,
|
||||
MemberCount: len(g.Members),
|
||||
PolicyCount: len(g.PolicyNames),
|
||||
Status: status,
|
||||
Members: g.Members,
|
||||
PolicyNames: g.PolicyNames,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) CreateGroup(ctx context.Context, name string) (*GroupData, error) {
|
||||
if s.credentialManager == nil {
|
||||
return nil, fmt.Errorf("credential manager not available")
|
||||
}
|
||||
|
||||
group := &iam_pb.Group{Name: name}
|
||||
if err := s.credentialManager.CreateGroup(ctx, group); err != nil {
|
||||
return nil, fmt.Errorf("failed to create group: %w", err)
|
||||
}
|
||||
glog.V(1).Infof("Created group %s", group.Name)
|
||||
return &GroupData{
|
||||
Name: group.Name,
|
||||
Status: "enabled",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteGroup(ctx context.Context, name string) error {
|
||||
if s.credentialManager == nil {
|
||||
return fmt.Errorf("credential manager not available")
|
||||
}
|
||||
// Check for members and attached policies before deleting (same guards as IAM handlers)
|
||||
g, err := s.credentialManager.GetGroup(ctx, name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get group: %w", err)
|
||||
}
|
||||
if len(g.Members) > 0 {
|
||||
return fmt.Errorf("cannot delete group %s: group has %d member(s): %w", name, len(g.Members), credential.ErrGroupNotEmpty)
|
||||
}
|
||||
if len(g.PolicyNames) > 0 {
|
||||
return fmt.Errorf("cannot delete group %s: group has %d attached policy(ies): %w", name, len(g.PolicyNames), credential.ErrGroupNotEmpty)
|
||||
}
|
||||
if err := s.credentialManager.DeleteGroup(ctx, name); err != nil {
|
||||
return fmt.Errorf("failed to delete group: %w", err)
|
||||
}
|
||||
glog.V(1).Infof("Deleted group %s", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) AddGroupMember(ctx context.Context, groupName, username string) error {
|
||||
if s.credentialManager == nil {
|
||||
return fmt.Errorf("credential manager not available")
|
||||
}
|
||||
g, err := s.credentialManager.GetGroup(ctx, groupName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get group: %w", err)
|
||||
}
|
||||
g = cloneGroup(g)
|
||||
if _, err := s.credentialManager.GetUser(ctx, username); err != nil {
|
||||
return fmt.Errorf("user %s not found: %w", username, err)
|
||||
}
|
||||
for _, m := range g.Members {
|
||||
if m == username {
|
||||
return nil // already a member
|
||||
}
|
||||
}
|
||||
g.Members = append(g.Members, username)
|
||||
if err := s.credentialManager.UpdateGroup(ctx, g); err != nil {
|
||||
return fmt.Errorf("failed to update group: %w", err)
|
||||
}
|
||||
glog.V(1).Infof("Added user %s to group %s", username, groupName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) RemoveGroupMember(ctx context.Context, groupName, username string) error {
|
||||
if s.credentialManager == nil {
|
||||
return fmt.Errorf("credential manager not available")
|
||||
}
|
||||
g, err := s.credentialManager.GetGroup(ctx, groupName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get group: %w", err)
|
||||
}
|
||||
g = cloneGroup(g)
|
||||
found := false
|
||||
var newMembers []string
|
||||
for _, m := range g.Members {
|
||||
if m == username {
|
||||
found = true
|
||||
} else {
|
||||
newMembers = append(newMembers, m)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("user %s is not a member of group %s: %w", username, groupName, credential.ErrUserNotInGroup)
|
||||
}
|
||||
g.Members = newMembers
|
||||
if err := s.credentialManager.UpdateGroup(ctx, g); err != nil {
|
||||
return fmt.Errorf("failed to update group: %w", err)
|
||||
}
|
||||
glog.V(1).Infof("Removed user %s from group %s", username, groupName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) AttachGroupPolicy(ctx context.Context, groupName, policyName string) error {
|
||||
if s.credentialManager == nil {
|
||||
return fmt.Errorf("credential manager not available")
|
||||
}
|
||||
g, err := s.credentialManager.GetGroup(ctx, groupName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get group: %w", err)
|
||||
}
|
||||
g = cloneGroup(g)
|
||||
if _, err := s.credentialManager.GetPolicy(ctx, policyName); err != nil {
|
||||
return fmt.Errorf("policy %s not found: %w", policyName, err)
|
||||
}
|
||||
for _, p := range g.PolicyNames {
|
||||
if p == policyName {
|
||||
return nil // already attached
|
||||
}
|
||||
}
|
||||
g.PolicyNames = append(g.PolicyNames, policyName)
|
||||
if err := s.credentialManager.UpdateGroup(ctx, g); err != nil {
|
||||
return fmt.Errorf("failed to update group: %w", err)
|
||||
}
|
||||
glog.V(1).Infof("Attached policy %s to group %s", policyName, groupName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) DetachGroupPolicy(ctx context.Context, groupName, policyName string) error {
|
||||
if s.credentialManager == nil {
|
||||
return fmt.Errorf("credential manager not available")
|
||||
}
|
||||
g, err := s.credentialManager.GetGroup(ctx, groupName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get group: %w", err)
|
||||
}
|
||||
g = cloneGroup(g)
|
||||
found := false
|
||||
var newPolicies []string
|
||||
for _, p := range g.PolicyNames {
|
||||
if p == policyName {
|
||||
found = true
|
||||
} else {
|
||||
newPolicies = append(newPolicies, p)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return fmt.Errorf("policy %s is not attached to group %s: %w", policyName, groupName, credential.ErrPolicyNotAttached)
|
||||
}
|
||||
g.PolicyNames = newPolicies
|
||||
if err := s.credentialManager.UpdateGroup(ctx, g); err != nil {
|
||||
return fmt.Errorf("failed to update group: %w", err)
|
||||
}
|
||||
glog.V(1).Infof("Detached policy %s from group %s", policyName, groupName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) SetGroupStatus(ctx context.Context, groupName string, enabled bool) error {
|
||||
if s.credentialManager == nil {
|
||||
return fmt.Errorf("credential manager not available")
|
||||
}
|
||||
g, err := s.credentialManager.GetGroup(ctx, groupName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get group: %w", err)
|
||||
}
|
||||
g = cloneGroup(g)
|
||||
g.Disabled = !enabled
|
||||
if err := s.credentialManager.UpdateGroup(ctx, g); err != nil {
|
||||
return fmt.Errorf("failed to update group: %w", err)
|
||||
}
|
||||
glog.V(1).Infof("Set group %s status to enabled=%v", groupName, enabled)
|
||||
return nil
|
||||
}
|
||||
@@ -324,7 +324,7 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
|
||||
}
|
||||
|
||||
// Only process directories that are versions (start with "v")
|
||||
if versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
|
||||
if versionResp.Entry != nil && versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
|
||||
versionDir := filepath.Join(topicDir, versionResp.Entry.Name)
|
||||
|
||||
// List all partition directories under the version directory (e.g., 0315-0630)
|
||||
@@ -352,7 +352,7 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
|
||||
}
|
||||
|
||||
// Only process directories that are partitions (format: NNNN-NNNN)
|
||||
if partitionResp.Entry.IsDirectory {
|
||||
if partitionResp.Entry != nil && partitionResp.Entry.IsDirectory {
|
||||
// Parse partition range to get partition start ID (e.g., "0315-0630" -> 315)
|
||||
var partitionStart, partitionStop int32
|
||||
if n, err := fmt.Sscanf(partitionResp.Entry.Name, "%04d-%04d", &partitionStart, &partitionStop); n != 2 || err != nil {
|
||||
@@ -387,11 +387,11 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
|
||||
}
|
||||
|
||||
// Only process .offset files
|
||||
if !offsetResp.Entry.IsDirectory && strings.HasSuffix(offsetResp.Entry.Name, ".offset") {
|
||||
if offsetResp.Entry != nil && !offsetResp.Entry.IsDirectory && strings.HasSuffix(offsetResp.Entry.Name, ".offset") {
|
||||
consumerGroup := strings.TrimSuffix(offsetResp.Entry.Name, ".offset")
|
||||
|
||||
// Read the offset value from the file
|
||||
offsetData, err := filer.ReadInsideFiler(client, partitionDir, offsetResp.Entry.Name)
|
||||
offsetData, err := filer.ReadInsideFiler(context.Background(), client, partitionDir, offsetResp.Entry.Name)
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to read offset file %s: %v", offsetResp.Entry.Name, err)
|
||||
continue
|
||||
@@ -401,7 +401,10 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
|
||||
offset := int64(util.BytesToUint64(offsetData))
|
||||
|
||||
// Get the file modification time
|
||||
lastUpdated := time.Unix(offsetResp.Entry.Attributes.Mtime, 0)
|
||||
var lastUpdated time.Time
|
||||
if offsetResp.Entry.Attributes != nil {
|
||||
lastUpdated = time.Unix(offsetResp.Entry.Attributes.Mtime, 0)
|
||||
}
|
||||
|
||||
offsets = append(offsets, ConsumerGroupOffsetInfo{
|
||||
ConsumerGroup: consumerGroup,
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -130,6 +131,47 @@ func (s *AdminServer) GetPluginJobDetailAPI(w http.ResponseWriter, r *http.Reque
|
||||
writeJSON(w, http.StatusOK, detail)
|
||||
}
|
||||
|
||||
// ExpirePluginJobAPI marks a job as failed so it no longer blocks scheduling.
|
||||
func (s *AdminServer) ExpirePluginJobAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobID := strings.TrimSpace(mux.Vars(r)["jobId"])
|
||||
if jobID == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "jobId is required")
|
||||
return
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Reason string `json:"reason"`
|
||||
}
|
||||
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil && err != io.EOF {
|
||||
writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
job, expired, err := s.ExpirePluginJob(jobID, req.Reason)
|
||||
if err != nil {
|
||||
if errors.Is(err, plugin.ErrJobNotFound) {
|
||||
writeJSONError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
response := map[string]interface{}{
|
||||
"job_id": jobID,
|
||||
"expired": expired,
|
||||
}
|
||||
if job != nil {
|
||||
response["job"] = job
|
||||
}
|
||||
if !expired {
|
||||
response["message"] = "job is not active"
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// GetPluginActivitiesAPI returns recent plugin activities.
|
||||
func (s *AdminServer) GetPluginActivitiesAPI(w http.ResponseWriter, r *http.Request) {
|
||||
query := r.URL.Query()
|
||||
@@ -172,6 +214,27 @@ func (s *AdminServer) GetPluginSchedulerStatesAPI(w http.ResponseWriter, r *http
|
||||
writeJSON(w, http.StatusOK, states)
|
||||
}
|
||||
|
||||
// GetPluginSchedulerStatusAPI returns scheduler status including in-process jobs and lock state.
|
||||
func (s *AdminServer) GetPluginSchedulerStatusAPI(w http.ResponseWriter, r *http.Request) {
|
||||
pluginSvc := s.GetPlugin()
|
||||
if pluginSvc == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"enabled": false,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
response := map[string]interface{}{
|
||||
"enabled": true,
|
||||
"scheduler": pluginSvc.GetSchedulerStatus(),
|
||||
}
|
||||
if s.pluginLock != nil {
|
||||
response["lock"] = s.pluginLock.Status()
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// RequestPluginJobTypeSchemaAPI asks a worker for one job type schema.
|
||||
func (s *AdminServer) RequestPluginJobTypeSchemaAPI(w http.ResponseWriter, r *http.Request) {
|
||||
jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
|
||||
@@ -235,6 +298,9 @@ func (s *AdminServer) GetPluginJobTypeConfigAPI(w http.ResponseWriter, r *http.R
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
|
||||
}
|
||||
}
|
||||
if descriptor, err := s.LoadPluginJobTypeDescriptor(jobType); err == nil && descriptor != nil {
|
||||
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
|
||||
}
|
||||
|
||||
renderProtoJSON(w, http.StatusOK, config)
|
||||
}
|
||||
@@ -413,6 +479,14 @@ func (s *AdminServer) RunPluginJobTypeAPI(w http.ResponseWriter, r *http.Request
|
||||
writeJSONError(w, http.StatusBadRequest, "jobType is required")
|
||||
return
|
||||
}
|
||||
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detect+execute %s", jobType))
|
||||
if err != nil {
|
||||
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if releaseLock != nil {
|
||||
defer releaseLock()
|
||||
}
|
||||
|
||||
var req struct {
|
||||
ClusterContext json.RawMessage `json:"cluster_context"`
|
||||
@@ -729,6 +803,93 @@ func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, i
|
||||
return jobSpec
|
||||
}
|
||||
|
||||
func applyDescriptorDefaultsToPersistedConfig(
|
||||
config *plugin_pb.PersistedJobTypeConfig,
|
||||
descriptor *plugin_pb.JobTypeDescriptor,
|
||||
) {
|
||||
if config == nil || descriptor == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if config.AdminConfigValues == nil {
|
||||
config.AdminConfigValues = map[string]*plugin_pb.ConfigValue{}
|
||||
}
|
||||
if config.WorkerConfigValues == nil {
|
||||
config.WorkerConfigValues = map[string]*plugin_pb.ConfigValue{}
|
||||
}
|
||||
if config.AdminRuntime == nil {
|
||||
config.AdminRuntime = &plugin_pb.AdminRuntimeConfig{}
|
||||
}
|
||||
|
||||
if descriptor.AdminConfigForm != nil {
|
||||
for key, value := range descriptor.AdminConfigForm.DefaultValues {
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
current := config.AdminConfigValues[key]
|
||||
if current == nil {
|
||||
config.AdminConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(descriptor.JobType, "admin_script") &&
|
||||
key == "script" &&
|
||||
isBlankStringConfigValue(current) {
|
||||
config.AdminConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
if descriptor.WorkerConfigForm != nil {
|
||||
for key, value := range descriptor.WorkerConfigForm.DefaultValues {
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
if config.WorkerConfigValues[key] != nil {
|
||||
continue
|
||||
}
|
||||
config.WorkerConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
|
||||
}
|
||||
}
|
||||
if descriptor.AdminRuntimeDefaults != nil {
|
||||
runtime := config.AdminRuntime
|
||||
defaults := descriptor.AdminRuntimeDefaults
|
||||
if runtime.DetectionIntervalSeconds <= 0 {
|
||||
runtime.DetectionIntervalSeconds = defaults.DetectionIntervalSeconds
|
||||
}
|
||||
if runtime.DetectionTimeoutSeconds <= 0 {
|
||||
runtime.DetectionTimeoutSeconds = defaults.DetectionTimeoutSeconds
|
||||
}
|
||||
if runtime.MaxJobsPerDetection <= 0 {
|
||||
runtime.MaxJobsPerDetection = defaults.MaxJobsPerDetection
|
||||
}
|
||||
if runtime.GlobalExecutionConcurrency <= 0 {
|
||||
runtime.GlobalExecutionConcurrency = defaults.GlobalExecutionConcurrency
|
||||
}
|
||||
if runtime.PerWorkerExecutionConcurrency <= 0 {
|
||||
runtime.PerWorkerExecutionConcurrency = defaults.PerWorkerExecutionConcurrency
|
||||
}
|
||||
if runtime.JobTypeMaxRuntimeSeconds <= 0 {
|
||||
runtime.JobTypeMaxRuntimeSeconds = defaults.JobTypeMaxRuntimeSeconds
|
||||
}
|
||||
if runtime.RetryBackoffSeconds <= 0 {
|
||||
runtime.RetryBackoffSeconds = defaults.RetryBackoffSeconds
|
||||
}
|
||||
if runtime.RetryLimit < 0 {
|
||||
runtime.RetryLimit = defaults.RetryLimit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isBlankStringConfigValue(value *plugin_pb.ConfigValue) bool {
|
||||
if value == nil {
|
||||
return true
|
||||
}
|
||||
kind, ok := value.Kind.(*plugin_pb.ConfigValue_StringValue)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(kind.StringValue) == ""
|
||||
}
|
||||
|
||||
func parsePositiveInt(raw string, defaultValue int) int {
|
||||
value, err := strconv.Atoi(strings.TrimSpace(raw))
|
||||
if err != nil || value <= 0 {
|
||||
|
||||
@@ -1,11 +1,120 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
|
||||
)
|
||||
|
||||
func TestExpirePluginJobAPI(t *testing.T) {
|
||||
makeRequest := func(adminServer *AdminServer, jobID string, body io.Reader) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/plugin/jobs/"+jobID+"/expire", body)
|
||||
req = mux.SetURLVars(req, map[string]string{"jobId": jobID})
|
||||
recorder := httptest.NewRecorder()
|
||||
adminServer.ExpirePluginJobAPI(recorder, req)
|
||||
return recorder
|
||||
}
|
||||
|
||||
t.Run("empty job id", func(t *testing.T) {
|
||||
recorder := makeRequest(&AdminServer{}, "", nil)
|
||||
if recorder.Code != http.StatusBadRequest {
|
||||
t.Fatalf("expected 400, got %d", recorder.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid json", func(t *testing.T) {
|
||||
recorder := makeRequest(&AdminServer{}, "job-id", strings.NewReader("{"))
|
||||
if recorder.Code != http.StatusBadRequest {
|
||||
t.Fatalf("expected 400, got %d", recorder.Code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("job not found", func(t *testing.T) {
|
||||
adminServer := &AdminServer{
|
||||
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
|
||||
return nil, false, plugin.ErrJobNotFound
|
||||
},
|
||||
}
|
||||
recorder := makeRequest(adminServer, "missing", strings.NewReader(`{"reason":"nope"}`))
|
||||
if recorder.Code != http.StatusNotFound {
|
||||
t.Fatalf("expected 404, got %d", recorder.Code)
|
||||
}
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("failed to unmarshal body: %v", err)
|
||||
}
|
||||
if payload["error"] == nil {
|
||||
t.Fatalf("expected error payload, got %v", payload)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("successful expire", func(t *testing.T) {
|
||||
expected := &plugin.TrackedJob{JobID: "foo", State: "assigned"}
|
||||
adminServer := &AdminServer{
|
||||
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
|
||||
if jobID != "foo" {
|
||||
return nil, false, errors.New("unexpected")
|
||||
}
|
||||
return expected, true, nil
|
||||
},
|
||||
}
|
||||
recorder := makeRequest(adminServer, "foo", strings.NewReader(`{"reason":"cleanup"}`))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", recorder.Code)
|
||||
}
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("failed to decode payload: %v", err)
|
||||
}
|
||||
if payload["job_id"] != "foo" {
|
||||
t.Fatalf("expected job_id foo, got %v", payload["job_id"])
|
||||
}
|
||||
if expired, ok := payload["expired"].(bool); !ok || !expired {
|
||||
t.Fatalf("expected expired=true, got %v", payload["expired"])
|
||||
}
|
||||
jobData, ok := payload["job"].(map[string]any)
|
||||
if !ok || jobData["job_id"] != "foo" {
|
||||
t.Fatalf("expected job info with job_id, got %v", payload["job"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-active job", func(t *testing.T) {
|
||||
adminServer := &AdminServer{
|
||||
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
|
||||
return nil, false, nil
|
||||
},
|
||||
}
|
||||
recorder := makeRequest(adminServer, "bar", strings.NewReader(`{"reason":"ignore"}`))
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("expected 200, got %d", recorder.Code)
|
||||
}
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("failed to decode payload: %v", err)
|
||||
}
|
||||
if payload["job_id"] != "bar" {
|
||||
t.Fatalf("expected job_id bar, got %v", payload["job_id"])
|
||||
}
|
||||
if expired, ok := payload["expired"].(bool); !ok || expired {
|
||||
t.Fatalf("expected expired=false, got %v", payload["expired"])
|
||||
}
|
||||
if payload["message"] != "job is not active" {
|
||||
t.Fatalf("expected message job is not active, got %v", payload["message"])
|
||||
}
|
||||
if _, exists := payload["job"]; exists {
|
||||
t.Fatalf("expected no job payload for non-active job, got %v", payload["job"])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -31,3 +140,83 @@ func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
|
||||
t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &plugin_pb.PersistedJobTypeConfig{
|
||||
JobType: "admin_script",
|
||||
AdminConfigValues: map[string]*plugin_pb.ConfigValue{},
|
||||
WorkerConfigValues: map[string]*plugin_pb.ConfigValue{},
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
|
||||
}
|
||||
descriptor := &plugin_pb.JobTypeDescriptor{
|
||||
JobType: "admin_script",
|
||||
AdminConfigForm: &plugin_pb.ConfigForm{
|
||||
DefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"script": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.balance -apply"},
|
||||
},
|
||||
"run_interval_minutes": {
|
||||
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 17},
|
||||
},
|
||||
},
|
||||
},
|
||||
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
|
||||
DetectionIntervalSeconds: 60,
|
||||
DetectionTimeoutSeconds: 300,
|
||||
},
|
||||
}
|
||||
|
||||
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
|
||||
|
||||
script := config.AdminConfigValues["script"]
|
||||
if script == nil {
|
||||
t.Fatalf("expected script default to be backfilled")
|
||||
}
|
||||
scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
|
||||
if !ok || scriptKind.StringValue == "" {
|
||||
t.Fatalf("expected non-empty script default, got=%+v", script)
|
||||
}
|
||||
if config.AdminRuntime.DetectionIntervalSeconds != 60 {
|
||||
t.Fatalf("expected runtime detection interval default to be backfilled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyDescriptorDefaultsToPersistedConfigReplacesBlankAdminScript(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
config := &plugin_pb.PersistedJobTypeConfig{
|
||||
JobType: "admin_script",
|
||||
AdminConfigValues: map[string]*plugin_pb.ConfigValue{
|
||||
"script": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: " "},
|
||||
},
|
||||
},
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
|
||||
}
|
||||
descriptor := &plugin_pb.JobTypeDescriptor{
|
||||
JobType: "admin_script",
|
||||
AdminConfigForm: &plugin_pb.ConfigForm{
|
||||
DefaultValues: map[string]*plugin_pb.ConfigValue{
|
||||
"script": {
|
||||
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.fix.replication -apply"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
|
||||
|
||||
script := config.AdminConfigValues["script"]
|
||||
if script == nil {
|
||||
t.Fatalf("expected script config value")
|
||||
}
|
||||
scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
|
||||
if !ok {
|
||||
t.Fatalf("expected string script config value, got=%T", script.Kind)
|
||||
}
|
||||
if scriptKind.StringValue != "volume.fix.replication -apply" {
|
||||
t.Fatalf("expected blank script to be replaced by default, got=%q", scriptKind.StringValue)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,17 +151,21 @@ func (p *TopicRetentionPurger) purgeTopicData(topicRetention TopicRetentionConfi
|
||||
}
|
||||
|
||||
// Only process directories that are versions (start with "v")
|
||||
if versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
|
||||
if versionResp.Entry != nil && versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
|
||||
versionTime, err := p.parseVersionTime(versionResp.Entry.Name)
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to parse version time from %s: %v", versionResp.Entry.Name, err)
|
||||
continue
|
||||
}
|
||||
|
||||
var modTime time.Time
|
||||
if versionResp.Entry.Attributes != nil {
|
||||
modTime = time.Unix(versionResp.Entry.Attributes.Mtime, 0)
|
||||
}
|
||||
versionDirs = append(versionDirs, VersionDirInfo{
|
||||
Name: versionResp.Entry.Name,
|
||||
VersionTime: versionTime,
|
||||
ModTime: time.Unix(versionResp.Entry.Attributes.Mtime, 0),
|
||||
ModTime: modTime,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -260,6 +264,9 @@ func (p *TopicRetentionPurger) deleteDirectoryRecursively(client filer_pb.Seawee
|
||||
return fmt.Errorf("failed to receive entries: %w", err)
|
||||
}
|
||||
|
||||
if resp.Entry == nil {
|
||||
continue
|
||||
}
|
||||
entryPath := filepath.Join(dirPath, resp.Entry.Name)
|
||||
|
||||
if resp.Entry.IsDirectory {
|
||||
|
||||
@@ -589,6 +589,30 @@ type UpdateServiceAccountRequest struct {
|
||||
Expiration string `json:"expiration,omitempty"`
|
||||
}
|
||||
|
||||
// Group management structures
|
||||
type GroupData struct {
|
||||
Name string `json:"name"`
|
||||
MemberCount int `json:"member_count"`
|
||||
PolicyCount int `json:"policy_count"`
|
||||
Status string `json:"status"` // "enabled" or "disabled"
|
||||
Members []string `json:"members"`
|
||||
PolicyNames []string `json:"policy_names"`
|
||||
}
|
||||
|
||||
type GroupsPageData struct {
|
||||
Username string `json:"username"`
|
||||
Groups []GroupData `json:"groups"`
|
||||
TotalGroups int `json:"total_groups"`
|
||||
ActiveGroups int `json:"active_groups"`
|
||||
AvailableUsers []string `json:"available_users"`
|
||||
AvailablePolicies []string `json:"available_policies"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
type CreateGroupRequest struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// STS Configuration display types
|
||||
type STSConfigData struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
|
||||
@@ -4,13 +4,21 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrAccessKeyInUse = errors.New("access key already in use")
|
||||
ErrUserNotFound = errors.New("user not found")
|
||||
ErrInvalidInput = errors.New("invalid input")
|
||||
)
|
||||
|
||||
// CreateObjectStoreUser creates a new user using the credential manager
|
||||
func (s *AdminServer) CreateObjectStoreUser(req CreateUserRequest) (*ObjectStoreUser, error) {
|
||||
if s.credentialManager == nil {
|
||||
@@ -187,6 +195,24 @@ func (s *AdminServer) GetObjectStoreUserDetails(username string) (*UserDetails,
|
||||
details.Email = identity.Account.EmailAddress
|
||||
}
|
||||
|
||||
// Look up groups the user belongs to
|
||||
groupNames, err := s.credentialManager.ListGroups(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list groups: %w", err)
|
||||
}
|
||||
for _, gName := range groupNames {
|
||||
g, err := s.credentialManager.GetGroup(ctx, gName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get group %s: %w", gName, err)
|
||||
}
|
||||
for _, member := range g.Members {
|
||||
if member == username {
|
||||
details.Groups = append(details.Groups, gName)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert credentials to access key info
|
||||
for _, cred := range identity.Credentials {
|
||||
details.AccessKeys = append(details.AccessKeys, AccessKeyInfo{
|
||||
@@ -201,7 +227,7 @@ func (s *AdminServer) GetObjectStoreUserDetails(username string) (*UserDetails,
|
||||
}
|
||||
|
||||
// CreateAccessKey creates a new access key for a user
|
||||
func (s *AdminServer) CreateAccessKey(username string) (*AccessKeyInfo, error) {
|
||||
func (s *AdminServer) CreateAccessKey(username string, req *CreateAccessKeyRequest) (*AccessKeyInfo, error) {
|
||||
if s.credentialManager == nil {
|
||||
return nil, fmt.Errorf("credential manager not available")
|
||||
}
|
||||
@@ -212,14 +238,41 @@ func (s *AdminServer) CreateAccessKey(username string) (*AccessKeyInfo, error) {
|
||||
_, err := s.credentialManager.GetUser(ctx, username)
|
||||
if err != nil {
|
||||
if err == credential.ErrUserNotFound {
|
||||
return nil, fmt.Errorf("user %s not found", username)
|
||||
return nil, fmt.Errorf("user %s: %w", username, ErrUserNotFound)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to get user: %w", err)
|
||||
}
|
||||
|
||||
// Generate new access key
|
||||
accessKey := generateAccessKey()
|
||||
secretKey := generateSecretKey()
|
||||
if req == nil {
|
||||
req = &CreateAccessKeyRequest{}
|
||||
}
|
||||
|
||||
// Validate provided keys
|
||||
if req.AccessKey != "" && (len(req.AccessKey) < 4 || len(req.AccessKey) > 128) {
|
||||
return nil, fmt.Errorf("access key must be between 4 and 128 characters: %w", ErrInvalidInput)
|
||||
}
|
||||
if req.SecretKey != "" && (len(req.SecretKey) < 8 || len(req.SecretKey) > 128) {
|
||||
return nil, fmt.Errorf("secret key must be between 8 and 128 characters: %w", ErrInvalidInput)
|
||||
}
|
||||
|
||||
// Use provided keys or generate new ones
|
||||
accessKey := req.AccessKey
|
||||
if accessKey == "" {
|
||||
accessKey = generateAccessKey()
|
||||
}
|
||||
secretKey := req.SecretKey
|
||||
if secretKey == "" {
|
||||
secretKey = generateSecretKey()
|
||||
}
|
||||
|
||||
// Verify access key is globally unique
|
||||
existingUser, err := s.credentialManager.GetUserByAccessKey(ctx, accessKey)
|
||||
if existingUser != nil {
|
||||
return nil, ErrAccessKeyInUse
|
||||
}
|
||||
if err != nil && !errors.Is(err, credential.ErrAccessKeyNotFound) && !isNotFoundError(err) {
|
||||
return nil, fmt.Errorf("failed to check access key uniqueness: %w", err)
|
||||
}
|
||||
|
||||
credential := &iam_pb.Credential{
|
||||
AccessKey: accessKey,
|
||||
@@ -364,6 +417,12 @@ func (s *AdminServer) UpdateUserPolicies(username string, actions []string) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
// isNotFoundError checks for "not found" in the error message as a fallback
|
||||
// for stores (e.g. gRPC) that don't return the credential.ErrAccessKeyNotFound sentinel.
|
||||
func isNotFoundError(err error) bool {
|
||||
return err != nil && strings.Contains(strings.ToLower(err.Error()), "not found")
|
||||
}
|
||||
|
||||
// Helper functions for generating keys and IDs
|
||||
func generateAccessKey() string {
|
||||
// Generate 20-character access key (AWS standard)
|
||||
|
||||
@@ -457,6 +457,7 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro
|
||||
|
||||
// Process disk information
|
||||
for _, diskInfo := range node.DiskInfos {
|
||||
vs.MaxVolumes += int(diskInfo.MaxVolumeCount)
|
||||
vs.DiskCapacity += int64(diskInfo.MaxVolumeCount) * int64(volumeSizeLimitMB) * 1024 * 1024 // Use actual volume size limit
|
||||
|
||||
// Count regular volumes and calculate disk usage
|
||||
|
||||
@@ -3,6 +3,7 @@ package handlers
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@@ -28,6 +29,7 @@ type AdminHandlers struct {
|
||||
pluginHandlers *PluginHandlers
|
||||
mqHandlers *MessageQueueHandlers
|
||||
serviceAccountHandlers *ServiceAccountHandlers
|
||||
groupHandlers *GroupHandlers
|
||||
}
|
||||
|
||||
// NewAdminHandlers creates a new instance of AdminHandlers
|
||||
@@ -40,6 +42,7 @@ func NewAdminHandlers(adminServer *dash.AdminServer, store sessions.Store) *Admi
|
||||
pluginHandlers := NewPluginHandlers(adminServer)
|
||||
mqHandlers := NewMessageQueueHandlers(adminServer)
|
||||
serviceAccountHandlers := NewServiceAccountHandlers(adminServer)
|
||||
groupHandlers := NewGroupHandlers(adminServer)
|
||||
return &AdminHandlers{
|
||||
adminServer: adminServer,
|
||||
sessionStore: store,
|
||||
@@ -51,6 +54,7 @@ func NewAdminHandlers(adminServer *dash.AdminServer, store sessions.Store) *Admi
|
||||
pluginHandlers: pluginHandlers,
|
||||
mqHandlers: mqHandlers,
|
||||
serviceAccountHandlers: serviceAccountHandlers,
|
||||
groupHandlers: groupHandlers,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,6 +108,7 @@ func (h *AdminHandlers) registerUIRoutes(r *mux.Router) {
|
||||
r.HandleFunc("/object-store/buckets/{bucket}", h.ShowBucketDetails).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/users", h.userHandlers.ShowObjectStoreUsers).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/policies", h.policyHandlers.ShowPolicies).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/groups", h.groupHandlers.ShowGroups).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/s3tables/buckets", h.ShowS3TablesBuckets).Methods(http.MethodGet)
|
||||
r.HandleFunc("/object-store/s3tables/buckets/{bucket}/namespaces", h.ShowS3TablesNamespaces).Methods(http.MethodGet)
|
||||
@@ -185,6 +190,19 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
|
||||
saApi.Handle("/{id}", wrapWrite(h.serviceAccountHandlers.UpdateServiceAccount)).Methods(http.MethodPut)
|
||||
saApi.Handle("/{id}", wrapWrite(h.serviceAccountHandlers.DeleteServiceAccount)).Methods(http.MethodDelete)
|
||||
|
||||
groupsApi := api.PathPrefix("/groups").Subrouter()
|
||||
groupsApi.HandleFunc("", h.groupHandlers.GetGroups).Methods(http.MethodGet)
|
||||
groupsApi.Handle("", wrapWrite(h.groupHandlers.CreateGroup)).Methods(http.MethodPost)
|
||||
groupsApi.HandleFunc("/{name}", h.groupHandlers.GetGroupDetails).Methods(http.MethodGet)
|
||||
groupsApi.Handle("/{name}", wrapWrite(h.groupHandlers.DeleteGroup)).Methods(http.MethodDelete)
|
||||
groupsApi.Handle("/{name}/status", wrapWrite(h.groupHandlers.SetGroupStatus)).Methods(http.MethodPut)
|
||||
groupsApi.HandleFunc("/{name}/members", h.groupHandlers.GetGroupMembers).Methods(http.MethodGet)
|
||||
groupsApi.Handle("/{name}/members", wrapWrite(h.groupHandlers.AddGroupMember)).Methods(http.MethodPost)
|
||||
groupsApi.Handle("/{name}/members/{username}", wrapWrite(h.groupHandlers.RemoveGroupMember)).Methods(http.MethodDelete)
|
||||
groupsApi.HandleFunc("/{name}/policies", h.groupHandlers.GetGroupPolicies).Methods(http.MethodGet)
|
||||
groupsApi.Handle("/{name}/policies", wrapWrite(h.groupHandlers.AttachGroupPolicy)).Methods(http.MethodPost)
|
||||
groupsApi.Handle("/{name}/policies/{policyName}", wrapWrite(h.groupHandlers.DetachGroupPolicy)).Methods(http.MethodDelete)
|
||||
|
||||
policyApi := api.PathPrefix("/object-store/policies").Subrouter()
|
||||
policyApi.HandleFunc("", h.policyHandlers.GetPolicies).Methods(http.MethodGet)
|
||||
policyApi.Handle("", wrapWrite(h.policyHandlers.CreatePolicy)).Methods(http.MethodPost)
|
||||
@@ -234,6 +252,7 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
|
||||
pluginApi.HandleFunc("/jobs/{jobId}/detail", h.adminServer.GetPluginJobDetailAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/activities", h.adminServer.GetPluginActivitiesAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/scheduler-states", h.adminServer.GetPluginSchedulerStatesAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/scheduler-status", h.adminServer.GetPluginSchedulerStatusAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/job-types/{jobType}/descriptor", h.adminServer.GetPluginJobTypeDescriptorAPI).Methods(http.MethodGet)
|
||||
pluginApi.HandleFunc("/job-types/{jobType}/schema", h.adminServer.RequestPluginJobTypeSchemaAPI).Methods(http.MethodPost)
|
||||
pluginApi.HandleFunc("/job-types/{jobType}/config", h.adminServer.GetPluginJobTypeConfigAPI).Methods(http.MethodGet)
|
||||
@@ -242,6 +261,7 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
|
||||
pluginApi.Handle("/job-types/{jobType}/detect", wrapWrite(h.adminServer.TriggerPluginDetectionAPI)).Methods(http.MethodPost)
|
||||
pluginApi.Handle("/job-types/{jobType}/run", wrapWrite(h.adminServer.RunPluginJobTypeAPI)).Methods(http.MethodPost)
|
||||
pluginApi.Handle("/jobs/execute", wrapWrite(h.adminServer.ExecutePluginJobAPI)).Methods(http.MethodPost)
|
||||
pluginApi.Handle("/jobs/{jobId}/expire", wrapWrite(h.adminServer.ExpirePluginJobAPI)).Methods(http.MethodPost)
|
||||
|
||||
mqApi := api.PathPrefix("/mq").Subrouter()
|
||||
mqApi.HandleFunc("/topics/{namespace}/{topic}", h.mqHandlers.GetTopicDetailsAPI).Methods(http.MethodGet)
|
||||
@@ -274,8 +294,26 @@ func (h *AdminHandlers) ShowDashboard(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// ShowS3Buckets renders the Object Store buckets management page
|
||||
func (h *AdminHandlers) ShowS3Buckets(w http.ResponseWriter, r *http.Request) {
|
||||
// Get Object Store buckets data from the server
|
||||
s3Data := h.getS3BucketsData(r)
|
||||
// Get pagination and sorting parameters from query string
|
||||
page := 1
|
||||
if p := r.URL.Query().Get("page"); p != "" {
|
||||
if parsed, err := strconv.Atoi(p); err == nil && parsed > 0 {
|
||||
page = parsed
|
||||
}
|
||||
}
|
||||
|
||||
pageSize := 100
|
||||
if ps := r.URL.Query().Get("pageSize"); ps != "" {
|
||||
if parsed, err := strconv.Atoi(ps); err == nil && parsed > 0 && parsed <= 1000 {
|
||||
pageSize = parsed
|
||||
}
|
||||
}
|
||||
|
||||
sortBy := defaultQuery(r.URL.Query().Get("sortBy"), "name")
|
||||
sortOrder := defaultQuery(r.URL.Query().Get("sortOrder"), "asc")
|
||||
|
||||
// Get Object Store buckets data with pagination
|
||||
s3Data := h.getS3BucketsData(r, page, pageSize, sortBy, sortOrder)
|
||||
username := h.getUsername(r)
|
||||
|
||||
// Render HTML template
|
||||
@@ -442,15 +480,15 @@ func (h *AdminHandlers) ShowBucketDetails(w http.ResponseWriter, r *http.Request
|
||||
writeJSON(w, http.StatusOK, details)
|
||||
}
|
||||
|
||||
// getS3BucketsData retrieves Object Store buckets data from the server
|
||||
func (h *AdminHandlers) getS3BucketsData(r *http.Request) dash.S3BucketsData {
|
||||
// getS3BucketsData retrieves Object Store buckets data from the server with pagination
|
||||
func (h *AdminHandlers) getS3BucketsData(r *http.Request, page, pageSize int, sortBy, sortOrder string) dash.S3BucketsData {
|
||||
username := dash.UsernameFromContext(r.Context())
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
|
||||
// Get Object Store buckets data
|
||||
data, err := h.adminServer.GetS3BucketsData()
|
||||
data, err := h.adminServer.GetS3BucketsData(page, pageSize, sortBy, sortOrder)
|
||||
if err != nil {
|
||||
// Return empty data on error
|
||||
return dash.S3BucketsData{
|
||||
@@ -459,6 +497,11 @@ func (h *AdminHandlers) getS3BucketsData(r *http.Request) dash.S3BucketsData {
|
||||
TotalBuckets: 0,
|
||||
TotalSize: 0,
|
||||
LastUpdated: time.Now(),
|
||||
CurrentPage: 1,
|
||||
TotalPages: 1,
|
||||
PageSize: pageSize,
|
||||
SortBy: sortBy,
|
||||
SortOrder: sortOrder,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,9 @@ func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_NoAuth(t *testing.T) {
|
||||
if !hasRoute(router, http.MethodGet, "/api/plugin/jobs/example/detail") {
|
||||
t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in no-auth mode")
|
||||
}
|
||||
if !hasRoute(router, http.MethodPost, "/api/plugin/jobs/example/expire") {
|
||||
t.Fatalf("expected POST /api/plugin/jobs/:jobId/expire to be registered in no-auth mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) {
|
||||
@@ -34,6 +37,9 @@ func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) {
|
||||
if !hasRoute(router, http.MethodGet, "/api/plugin/jobs/example/detail") {
|
||||
t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in auth mode")
|
||||
}
|
||||
if !hasRoute(router, http.MethodPost, "/api/plugin/jobs/example/expire") {
|
||||
t.Fatalf("expected POST /api/plugin/jobs/:jobId/expire to be registered in auth mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetupRoutes_RegistersPluginPages_NoAuth(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
|
||||
"github.com/seaweedfs/seaweedfs/weed/credential"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func groupErrorToHTTPStatus(err error) int {
|
||||
if errors.Is(err, credential.ErrGroupNotFound) {
|
||||
return http.StatusNotFound
|
||||
}
|
||||
if errors.Is(err, credential.ErrGroupAlreadyExists) {
|
||||
return http.StatusConflict
|
||||
}
|
||||
if errors.Is(err, credential.ErrUserNotInGroup) {
|
||||
return http.StatusBadRequest
|
||||
}
|
||||
if errors.Is(err, credential.ErrPolicyNotAttached) {
|
||||
return http.StatusBadRequest
|
||||
}
|
||||
if errors.Is(err, credential.ErrUserNotFound) {
|
||||
return http.StatusNotFound
|
||||
}
|
||||
if errors.Is(err, credential.ErrPolicyNotFound) {
|
||||
return http.StatusNotFound
|
||||
}
|
||||
if errors.Is(err, credential.ErrGroupNotEmpty) {
|
||||
return http.StatusConflict
|
||||
}
|
||||
return http.StatusInternalServerError
|
||||
}
|
||||
|
||||
type GroupHandlers struct {
|
||||
adminServer *dash.AdminServer
|
||||
}
|
||||
|
||||
func NewGroupHandlers(adminServer *dash.AdminServer) *GroupHandlers {
|
||||
return &GroupHandlers{adminServer: adminServer}
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) ShowGroups(w http.ResponseWriter, r *http.Request) {
|
||||
data, err := h.getGroupsPageData(r)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get groups data: %v", err)
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to load groups: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
component := app.Groups(data)
|
||||
viewCtx := layout.NewViewContext(r, dash.UsernameFromContext(r.Context()), dash.CSRFTokenFromContext(r.Context()))
|
||||
layoutComponent := layout.Layout(viewCtx, component)
|
||||
if err := layoutComponent.Render(r.Context(), &buf); err != nil {
|
||||
glog.Errorf("Failed to render groups template: %v", err)
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
_, _ = w.Write(buf.Bytes())
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) GetGroups(w http.ResponseWriter, r *http.Request) {
|
||||
groups, err := h.adminServer.GetGroups(r.Context())
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get groups: %v", err)
|
||||
writeJSONError(w, http.StatusInternalServerError, "Failed to get groups")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"groups": groups})
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) CreateGroup(w http.ResponseWriter, r *http.Request) {
|
||||
var req dash.CreateGroupRequest
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.Name == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "Group name is required")
|
||||
return
|
||||
}
|
||||
group, err := h.adminServer.CreateGroup(r.Context(), req.Name)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to create group: %v", err)
|
||||
writeJSONError(w, groupErrorToHTTPStatus(err), "Failed to create group: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, group)
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) GetGroupDetails(w http.ResponseWriter, r *http.Request) {
|
||||
name := mux.Vars(r)["name"]
|
||||
group, err := h.adminServer.GetGroupDetails(r.Context(), name)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get group details: %v", err)
|
||||
status := groupErrorToHTTPStatus(err)
|
||||
msg := "Failed to retrieve group"
|
||||
if status == http.StatusNotFound {
|
||||
msg = "Group not found"
|
||||
}
|
||||
writeJSONError(w, status, msg)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, group)
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) DeleteGroup(w http.ResponseWriter, r *http.Request) {
|
||||
name := mux.Vars(r)["name"]
|
||||
if err := h.adminServer.DeleteGroup(r.Context(), name); err != nil {
|
||||
glog.Errorf("Failed to delete group: %v", err)
|
||||
writeJSONError(w, groupErrorToHTTPStatus(err), "Failed to delete group: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"message": "Group deleted successfully"})
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) GetGroupMembers(w http.ResponseWriter, r *http.Request) {
|
||||
name := mux.Vars(r)["name"]
|
||||
group, err := h.adminServer.GetGroupDetails(r.Context(), name)
|
||||
if err != nil {
|
||||
writeJSONError(w, groupErrorToHTTPStatus(err), "Failed to get group: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"members": group.Members})
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) AddGroupMember(w http.ResponseWriter, r *http.Request) {
|
||||
name := mux.Vars(r)["name"]
|
||||
var req struct {
|
||||
Username string `json:"username"`
|
||||
}
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.Username == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "Username is required")
|
||||
return
|
||||
}
|
||||
if err := h.adminServer.AddGroupMember(r.Context(), name, req.Username); err != nil {
|
||||
writeJSONError(w, groupErrorToHTTPStatus(err), "Failed to add member: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"message": "Member added successfully"})
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) RemoveGroupMember(w http.ResponseWriter, r *http.Request) {
|
||||
name := mux.Vars(r)["name"]
|
||||
username := mux.Vars(r)["username"]
|
||||
if err := h.adminServer.RemoveGroupMember(r.Context(), name, username); err != nil {
|
||||
writeJSONError(w, groupErrorToHTTPStatus(err), "Failed to remove member: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"message": "Member removed successfully"})
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) GetGroupPolicies(w http.ResponseWriter, r *http.Request) {
|
||||
name := mux.Vars(r)["name"]
|
||||
group, err := h.adminServer.GetGroupDetails(r.Context(), name)
|
||||
if err != nil {
|
||||
writeJSONError(w, groupErrorToHTTPStatus(err), "Failed to get group: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{"policies": group.PolicyNames})
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) AttachGroupPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
name := mux.Vars(r)["name"]
|
||||
var req struct {
|
||||
PolicyName string `json:"policy_name"`
|
||||
}
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.PolicyName == "" {
|
||||
writeJSONError(w, http.StatusBadRequest, "Policy name is required")
|
||||
return
|
||||
}
|
||||
if err := h.adminServer.AttachGroupPolicy(r.Context(), name, req.PolicyName); err != nil {
|
||||
writeJSONError(w, groupErrorToHTTPStatus(err), "Failed to attach policy: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"message": "Policy attached successfully"})
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) DetachGroupPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
name := mux.Vars(r)["name"]
|
||||
policyName := mux.Vars(r)["policyName"]
|
||||
if err := h.adminServer.DetachGroupPolicy(r.Context(), name, policyName); err != nil {
|
||||
writeJSONError(w, groupErrorToHTTPStatus(err), "Failed to detach policy: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"message": "Policy detached successfully"})
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
|
||||
name := mux.Vars(r)["name"]
|
||||
var req struct {
|
||||
Enabled *bool `json:"enabled"`
|
||||
}
|
||||
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.Enabled == nil {
|
||||
writeJSONError(w, http.StatusBadRequest, "enabled field is required")
|
||||
return
|
||||
}
|
||||
if err := h.adminServer.SetGroupStatus(r.Context(), name, *req.Enabled); err != nil {
|
||||
writeJSONError(w, groupErrorToHTTPStatus(err), "Failed to update group status: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"message": "Group status updated"})
|
||||
}
|
||||
|
||||
func (h *GroupHandlers) getGroupsPageData(r *http.Request) (dash.GroupsPageData, error) {
|
||||
username := dash.UsernameFromContext(r.Context())
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
|
||||
groups, err := h.adminServer.GetGroups(r.Context())
|
||||
if err != nil {
|
||||
return dash.GroupsPageData{}, err
|
||||
}
|
||||
|
||||
activeCount := 0
|
||||
for _, g := range groups {
|
||||
if g.Status == "enabled" {
|
||||
activeCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Get available users for dropdown
|
||||
var availableUsers []string
|
||||
users, err := h.adminServer.GetObjectStoreUsers(r.Context())
|
||||
if err == nil {
|
||||
for _, user := range users {
|
||||
availableUsers = append(availableUsers, user.Username)
|
||||
}
|
||||
}
|
||||
|
||||
// Get available policies for dropdown
|
||||
var availablePolicies []string
|
||||
policies, err := h.adminServer.GetPolicies()
|
||||
if err == nil {
|
||||
for _, p := range policies {
|
||||
availablePolicies = append(availablePolicies, p.Name)
|
||||
}
|
||||
}
|
||||
|
||||
return dash.GroupsPageData{
|
||||
Username: username,
|
||||
Groups: groups,
|
||||
TotalGroups: len(groups),
|
||||
ActiveGroups: activeCount,
|
||||
AvailableUsers: availableUsers,
|
||||
AvailablePolicies: availablePolicies,
|
||||
LastUpdated: time.Now(),
|
||||
}, nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user