mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-06 14:45:51 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
330bd92ddc | ||
|
|
ba8816e2e1 | ||
|
|
3b9e367c1a | ||
|
|
ff5a8f0579 | ||
|
|
345ac950b6 | ||
|
|
9638d37fe2 | ||
|
|
fca1216f6d | ||
|
|
97e33b3dbd | ||
|
|
c50aaa9fbd | ||
|
|
a992417058 | ||
|
|
723fca35d4 | ||
|
|
f23e09f58b | ||
|
|
621834d96a | ||
|
|
2ee6e4f391 | ||
|
|
fe6f8d737d | ||
|
|
79722bcf30 | ||
|
|
b2b0a38e71 | ||
|
|
6a9e7360df | ||
|
|
f1e27b8f30 | ||
|
|
9e15823855 | ||
|
|
6940b7d06e | ||
|
|
23c25379ca | ||
|
|
c7c2d8d606 | ||
|
|
88c27615c4 | ||
|
|
49c66bbb2e | ||
|
|
94e0b902f9 | ||
|
|
3b05efbdbc | ||
|
|
b91427c30f | ||
|
|
550a4ff761 | ||
|
|
8b61fd77b5 | ||
|
|
4d513a2b3d | ||
|
|
c52d3d1229 | ||
|
|
9e575822a3 | ||
|
|
d399113e0c | ||
|
|
c106532b79 | ||
|
|
a4217dff5f | ||
|
|
745a7e40a6 | ||
|
|
d5ce6a4cda | ||
|
|
fe66d00ab0 | ||
|
|
2f155ee5ee | ||
|
|
549b65785d | ||
|
|
590e7efbef | ||
|
|
78c00e313a | ||
|
|
f5d26b803b | ||
|
|
25b0f86bda | ||
|
|
b049e883e1 | ||
|
|
c99e8d4152 | ||
|
|
3dcaee56aa | ||
|
|
21584e4ac8 | ||
|
|
01da26fbcb | ||
|
|
2c45b69775 | ||
|
|
b7bba7e7dc | ||
|
|
e7b2869aa9 | ||
|
|
bea0f8eda0 | ||
|
|
cf5043a9f9 | ||
|
|
5c8de5e282 | ||
|
|
ee468749bd | ||
|
|
08bd1e2563 | ||
|
|
8eee6b2a0e | ||
|
|
fe856928c4 | ||
|
|
6658a655f6 | ||
|
|
c5eadadf5a | ||
|
|
1e18c01a78 | ||
|
|
3e8d2a0a71 | ||
|
|
dbf6465b0e | ||
|
|
a27f6527ab | ||
|
|
0b41ade726 | ||
|
|
41e799b4e0 | ||
|
|
ee3d779a5d | ||
|
|
169ee629fa | ||
|
|
fb8390c6a7 | ||
|
|
191a858e72 | ||
|
|
fb4fb8b082 | ||
|
|
b1d7f3d6e8 | ||
|
|
e0da63fd0a | ||
|
|
2d556ac2a5 | ||
|
|
e862888d2d | ||
|
|
b142689232 | ||
|
|
473e699368 | ||
|
|
0115e60919 | ||
|
|
a6c3e96f7b | ||
|
|
dffe038efa | ||
|
|
4d4af0589b | ||
|
|
d98e104dc5 | ||
|
|
f5d71008d7 | ||
|
|
0d65daad4c | ||
|
|
612eae9ae8 | ||
|
|
a689c1e052 | ||
|
|
d4ebafbacd | ||
|
|
43aebc10da | ||
|
|
9d54f4d160 | ||
|
|
c8cfbaa069 | ||
|
|
1697ec862f | ||
|
|
1fdd9c3372 | ||
|
|
78a007d42b | ||
|
|
ef0bae45e3 | ||
|
|
32fade010a | ||
|
|
2f4cee9538 | ||
|
|
090d473822 | ||
|
|
d6f6bf4ce7 | ||
|
|
e4e4dea140 | ||
|
|
dfdace9a13 | ||
|
|
2b2ff008cd | ||
|
|
f17ec59d46 | ||
|
|
12c1190a5c | ||
|
|
922b0c3171 | ||
|
|
31867b6f75 | ||
|
|
e381b81b47 | ||
|
|
51ef39fc76 | ||
|
|
89b85bfd5e | ||
|
|
783fe25eac | ||
|
|
5c43f1e6a4 | ||
|
|
5eed1874a9 | ||
|
|
ae7743304c | ||
|
|
babf1b06ac | ||
|
|
6ff683a627 | ||
|
|
1fb3aefa95 | ||
|
|
47ef8c3cce | ||
|
|
f83cef1259 | ||
|
|
ab6351e1c9 | ||
|
|
9f1dd57939 | ||
|
|
1f70d82170 | ||
|
|
6d01e42cef | ||
|
|
5cea00ff07 | ||
|
|
d8c7c16aad | ||
|
|
b7c1eba0a1 | ||
|
|
ae19621230 | ||
|
|
f13e250fc3 | ||
|
|
dc4c62e742 | ||
|
|
a3af5eb77a | ||
|
|
1c0d37e15a | ||
|
|
6fc170c645 | ||
|
|
62a1178a0b | ||
|
|
da15ee3e49 | ||
|
|
b4d8350936 | ||
|
|
1d1634c2a2 | ||
|
|
44f580c24e | ||
|
|
04514071a7 | ||
|
|
580c2b4ad4 | ||
|
|
c62a332c7f | ||
|
|
2c551dad5d | ||
|
|
01c17478ae | ||
|
|
07002cf54c | ||
|
|
3aace37cf6 | ||
|
|
33da87452b | ||
|
|
08ee4e37d8 | ||
|
|
b30631c3b5 | ||
|
|
6cdd34da77 | ||
|
|
b01504649d | ||
|
|
24c78d524c | ||
|
|
05c184b610 | ||
|
|
33c1a8251a | ||
|
|
96a6e4c551 | ||
|
|
f4e472d396 | ||
|
|
fb0e12e985 | ||
|
|
1b9c8b8614 | ||
|
|
450407fda1 | ||
|
|
b09d4d5d69 | ||
|
|
ef3873b616 | ||
|
|
dd7003a328 | ||
|
|
22a9fbf062 | ||
|
|
06f90028f7 | ||
|
|
3b1920cf43 | ||
|
|
0dcb175514 | ||
|
|
ddcdefb6b4 | ||
|
|
d66e194284 | ||
|
|
6a12438351 | ||
|
|
b1b922d757 | ||
|
|
0be47f9efd | ||
|
|
62fd4bd017 | ||
|
|
09bb90e8dc | ||
|
|
d730f81321 | ||
|
|
6542d1e0aa | ||
|
|
e86e65e5ab | ||
|
|
8e9c12e2c7 | ||
|
|
4f5f1f6be7 | ||
|
|
20952aa514 |
@@ -15,15 +15,8 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
platform: [amd64, arm64, arm, 386]
|
||||
include:
|
||||
- platform: amd64
|
||||
qemu: false
|
||||
- platform: arm64
|
||||
qemu: true
|
||||
- platform: arm
|
||||
qemu: true
|
||||
- platform: 386
|
||||
qemu: true
|
||||
variant: [standard, large_disk]
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -43,6 +36,18 @@ jobs:
|
||||
[ -d /go/pkg ] && rm -rf /go/pkg || true
|
||||
echo "Available disk space after cleanup:"
|
||||
df -h
|
||||
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
echo "build_args=TAGS=5BytesOffset" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
echo "build_args=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
@@ -50,13 +55,13 @@ jobs:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=latest
|
||||
tags: type=raw,value=latest,suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
labels: |
|
||||
org.opencontainers.image.title=seaweedfs
|
||||
org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
- name: Set up QEMU
|
||||
if: matrix.qemu
|
||||
if: matrix.platform != 'amd64'
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Create BuildKit config
|
||||
run: |
|
||||
@@ -82,7 +87,7 @@ jobs:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
- name: Build ${{ matrix.platform }}
|
||||
- name: Build ${{ matrix.platform }} ${{ matrix.variant }}
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILDKIT: 1
|
||||
@@ -92,13 +97,14 @@ jobs:
|
||||
file: ./docker/Dockerfile.go_build
|
||||
platforms: linux/${{ matrix.platform }}
|
||||
# Push to GHCR only during build to avoid Docker Hub rate limits
|
||||
tags: ghcr.io/chrislusf/seaweedfs:latest-${{ matrix.platform }}
|
||||
tags: ghcr.io/chrislusf/seaweedfs:latest${{ steps.config.outputs.tag_suffix }}-${{ matrix.platform }}
|
||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.platform }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.variant }}-${{ matrix.platform }}
|
||||
build-args: |
|
||||
BUILDKIT_INLINE_CACHE=1
|
||||
BRANCH=${{ github.sha }}
|
||||
${{ steps.config.outputs.build_args }}
|
||||
- name: Clean up build artifacts
|
||||
if: always()
|
||||
run: |
|
||||
@@ -111,9 +117,22 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
if: github.event_name != 'pull_request'
|
||||
strategy:
|
||||
matrix:
|
||||
variant: [standard, large_disk]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Configure variant
|
||||
id: config
|
||||
run: |
|
||||
if [ "${{ matrix.variant }}" == "large_disk" ]; then
|
||||
echo "tag_suffix=_large_disk" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "tag_suffix=" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@v5
|
||||
@@ -121,7 +140,7 @@ jobs:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
ghcr.io/chrislusf/seaweedfs
|
||||
tags: type=raw,value=latest
|
||||
tags: type=raw,value=latest,suffix=${{ steps.config.outputs.tag_suffix }}
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
@@ -142,13 +161,15 @@ jobs:
|
||||
crane version
|
||||
- name: Create and push manifest
|
||||
run: |
|
||||
SUFFIX="${{ steps.config.outputs.tag_suffix }}"
|
||||
|
||||
# Create manifest on GHCR first (no rate limits)
|
||||
echo "Creating GHCR manifest (no rate limits)..."
|
||||
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:latest \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-386
|
||||
docker buildx imagetools create -t ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-386
|
||||
|
||||
# Copy the complete multi-arch image from GHCR to Docker Hub
|
||||
# This only requires one pull from GHCR (no rate limit) and one push to Docker Hub
|
||||
@@ -184,16 +205,16 @@ jobs:
|
||||
# Use crane or skopeo to copy, fallback to docker if not available
|
||||
if command -v crane &> /dev/null; then
|
||||
echo "Using crane to copy..."
|
||||
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:latest chrislusf/seaweedfs:latest
|
||||
retry_with_backoff crane copy ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} chrislusf/seaweedfs:latest${SUFFIX}
|
||||
elif command -v skopeo &> /dev/null; then
|
||||
echo "Using skopeo to copy..."
|
||||
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:latest docker://chrislusf/seaweedfs:latest
|
||||
retry_with_backoff skopeo copy --all docker://ghcr.io/chrislusf/seaweedfs:latest${SUFFIX} docker://chrislusf/seaweedfs:latest${SUFFIX}
|
||||
else
|
||||
echo "Using docker buildx imagetools (pulling 4 images from Docker Hub)..."
|
||||
# Fallback: create manifest directly on Docker Hub (pulls from Docker Hub - rate limited)
|
||||
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:latest \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:latest-386
|
||||
retry_with_backoff docker buildx imagetools create -t chrislusf/seaweedfs:latest${SUFFIX} \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-amd64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm64 \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-arm \
|
||||
ghcr.io/chrislusf/seaweedfs:latest${SUFFIX}-386
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
name: "S3 Tables Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-tables-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
jobs:
|
||||
s3-tables-tests:
|
||||
name: S3 Tables Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false ./weed
|
||||
|
||||
- name: Run S3 Tables Integration Tests
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3tables
|
||||
run: |
|
||||
set -x
|
||||
set -o pipefail
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting S3 Tables Tests ==="
|
||||
|
||||
# Run S3 Tables integration tests
|
||||
go test -v -timeout 20m . 2>&1 | tee test-output.log || {
|
||||
echo "S3 Tables integration tests failed"
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show test output on failure
|
||||
if: failure()
|
||||
working-directory: test/s3tables
|
||||
run: |
|
||||
echo "=== Test Output ==="
|
||||
if [ -f test-output.log ]; then
|
||||
tail -200 test-output.log
|
||||
fi
|
||||
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-tables-test-logs
|
||||
path: test/s3tables/test-output.log
|
||||
retention-days: 3
|
||||
|
||||
s3-tables-build-verification:
|
||||
name: S3 Tables Build Verification
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Verify S3 Tables Package Builds
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Building S3 Tables package ==="
|
||||
go build ./weed/s3api/s3tables || {
|
||||
echo "S3 Tables package build failed"
|
||||
exit 1
|
||||
}
|
||||
echo "S3 Tables package built successfully"
|
||||
|
||||
- name: Verify S3 API Integration Builds
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Building S3 API with S3 Tables integration ==="
|
||||
go build ./weed/s3api || {
|
||||
echo "S3 API build with S3 Tables failed"
|
||||
exit 1
|
||||
}
|
||||
echo "S3 API with S3 Tables integration built successfully"
|
||||
|
||||
- name: Run Go Tests for S3 Tables Package
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running Go unit tests for S3 Tables ==="
|
||||
go test -v -race -timeout 5m ./weed/s3api/s3tables/... || {
|
||||
echo "S3 Tables unit tests failed"
|
||||
exit 1
|
||||
}
|
||||
echo "S3 Tables unit tests passed"
|
||||
|
||||
s3-tables-fmt-check:
|
||||
name: S3 Tables Format Check
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Check Go Format
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Checking S3 Tables Go format ==="
|
||||
unformatted=$(gofmt -l ./weed/s3api/s3tables)
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "Go format check failed - files need formatting"
|
||||
echo "$unformatted"
|
||||
exit 1
|
||||
fi
|
||||
echo "All S3 Tables files are properly formatted"
|
||||
|
||||
- name: Check S3 Tables Test Format
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Checking S3 Tables test format ==="
|
||||
unformatted=$(gofmt -l ./test/s3tables)
|
||||
if [ -n "$unformatted" ]; then
|
||||
echo "Go format check failed for tests"
|
||||
echo "$unformatted"
|
||||
exit 1
|
||||
fi
|
||||
echo "All S3 Tables test files are properly formatted"
|
||||
|
||||
s3-tables-vet:
|
||||
name: S3 Tables Go Vet Check
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Run Go Vet
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running go vet on S3 Tables package ==="
|
||||
go vet ./weed/s3api/s3tables/... || {
|
||||
echo "go vet check failed"
|
||||
exit 1
|
||||
}
|
||||
echo "go vet checks passed"
|
||||
|
||||
- name: Run Go Vet on Tests
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running go vet on S3 Tables tests ==="
|
||||
go vet ./test/s3tables/... || {
|
||||
echo "go vet check failed for tests"
|
||||
exit 1
|
||||
}
|
||||
echo "go vet checks passed for tests"
|
||||
@@ -141,3 +141,4 @@ test/s3/iam/.test_env
|
||||
/test/erasure_coding/admin_dockertest/tmp
|
||||
/test/erasure_coding/admin_dockertest/task_logs
|
||||
weed_bin
|
||||
.aider*
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# Put this file to one of the location, with descending priority
|
||||
# ./security.toml
|
||||
# $HOME/.seaweedfs/security.toml
|
||||
# /etc/seaweedfs/security.toml
|
||||
# this file is read by master, volume server, filer, and worker
|
||||
|
||||
# comma separated origins allowed to make requests to the filer and s3 gateway.
|
||||
# enter in this format: https://domain.com, or http://localhost:port
|
||||
[cors.allowed_origins]
|
||||
values = "*"
|
||||
|
||||
# this jwt signing key is read by master and volume server, and it is used for write operations:
|
||||
# - the Master server generates the JWT, which can be used to write a certain file on a volume server
|
||||
# - the Volume server validates the JWT on writing
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
# [jwt.signing]
|
||||
# key = "V1JJVEVTRUNSRVRFWEFNUExFMTIzNDU2Nzg5MDEy" # Example: WRITESECRETEXAMPLE123456789012
|
||||
# this jwt signing key is read by master and volume server, and it is used for read operations:
|
||||
# - the Master server generates the JWT, which can be used to read a certain file on a volume server
|
||||
# - the Volume server validates the JWT on reading
|
||||
# [jwt.signing.read]
|
||||
# key = "UkVBRFNFQ1JFVUVYQU1QTEUxMjM0NTY3ODkwMTI=" # Example: READSECRETEXAMPLE123456789012
|
||||
# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
|
||||
# - f.e. the S3 API Shim generates the JWT
|
||||
# - the Filer server validates the JWT on writing
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
# [jwt.filer_signing]
|
||||
# key = "RklMRVJXUklURVNFQ1JFVEVYQU1QTEUxMjM0NTY3OA==" # Example: FILERWRITESECRETEXAMPLE12345678
|
||||
# If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
|
||||
# - f.e. the S3 API Shim generates the JWT
|
||||
# - the Filer server validates the JWT on reading
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
# [jwt.filer_signing.read]
|
||||
# key = "RklMRVJSRUFEU0VDUkVURVhBTVBMRTEyMzQ1Njc4OQ==" # Example: FILERREADSECRETEXAMPLE123456789
|
||||
@@ -5,7 +5,7 @@ go 1.24.9
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.50.1
|
||||
cloud.google.com/go/storage v1.59.1
|
||||
cloud.google.com/go/storage v1.59.2
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -46,7 +46,7 @@ require (
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.2
|
||||
github.com/klauspost/compress v1.18.3
|
||||
github.com/klauspost/reedsolomon v1.13.0
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.3
|
||||
@@ -133,13 +133,13 @@ require (
|
||||
github.com/gin-contrib/sessions v1.0.4
|
||||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.12
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.17.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/lib/pq v1.11.1
|
||||
github.com/linkedin/goavro/v2 v2.14.1
|
||||
github.com/mattn/go-sqlite3 v1.14.33
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
@@ -152,7 +152,7 @@ require (
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1
|
||||
github.com/shirou/gopsutil/v4 v4.25.12
|
||||
github.com/shirou/gopsutil/v4 v4.26.1
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.1
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
@@ -254,7 +254,7 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.2-0.20251110135918-10b7b7e7cd26 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
||||
@@ -349,7 +349,7 @@ require (
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
github.com/gorilla/sessions v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-hclog v1.6.3 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
@@ -458,11 +458,11 @@ require (
|
||||
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/term v0.39.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
|
||||
@@ -477,8 +477,8 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq
|
||||
cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc=
|
||||
cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
|
||||
cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y=
|
||||
cloud.google.com/go/storage v1.59.1 h1:DXAZLcTimtiXdGqDSnebROVPd9QvRsFVVlptz02Wk58=
|
||||
cloud.google.com/go/storage v1.59.1/go.mod h1:cMWbtM+anpC74gn6qjLh+exqYcfmB9Hqe5z6adx+CLI=
|
||||
cloud.google.com/go/storage v1.59.2 h1:gmOAuG1opU8YvycMNpP+DvHfT9BfzzK5Cy+arP+Nocw=
|
||||
cloud.google.com/go/storage v1.59.2/go.mod h1:cMWbtM+anpC74gn6qjLh+exqYcfmB9Hqe5z6adx+CLI=
|
||||
cloud.google.com/go/storagetransfer v1.5.0/go.mod h1:dxNzUopWy7RQevYFHewchb29POFv3/AaBgnhqzqiK0w=
|
||||
cloud.google.com/go/storagetransfer v1.6.0/go.mod h1:y77xm4CQV/ZhFZH75PLEXY0ROiS7Gh6pSKrM8dJyg6I=
|
||||
cloud.google.com/go/storagetransfer v1.7.0/go.mod h1:8Giuj1QNb1kfLAiWM1bN6dHzfdlDAVC9rv9abHot2W4=
|
||||
@@ -560,8 +560,8 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATV
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
|
||||
github.com/Azure/go-ntlmssp v0.0.2-0.20251110135918-10b7b7e7cd26 h1:gy/jrlpp8EfSyA73a51fofoSfhp5rPNQAUvDr4Dm91c=
|
||||
github.com/Azure/go-ntlmssp v0.0.2-0.20251110135918-10b7b7e7cd26/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A=
|
||||
github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM=
|
||||
github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
|
||||
@@ -1046,8 +1046,8 @@ github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w
|
||||
github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
|
||||
@@ -1206,8 +1206,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 h1:X5VWvz21y3gzm9Nw/kaUeku/1+uBhcekkmy4IkffJww=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1/go.mod h1:Zanoh4+gvIgluNqcfMVTJueD4wSS5hT7zTt4Mrutd90=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
@@ -1348,8 +1348,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
|
||||
github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
@@ -1385,8 +1385,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
|
||||
github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
|
||||
github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkFS2iI=
|
||||
github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
|
||||
github.com/linxGnu/grocksdb v1.10.3 h1:0laII9AQ6kFxo5SjhdTfSh9EgF20piD6TMHK6YuDm+4=
|
||||
@@ -1658,8 +1658,8 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/shirou/gopsutil/v4 v4.25.12 h1:e7PvW/0RmJ8p8vPGJH4jvNkOyLmbkXgXW4m6ZPic6CY=
|
||||
github.com/shirou/gopsutil/v4 v4.25.12/go.mod h1:EivAfP5x2EhLp2ovdpKSozecVXn1TmuG7SMzs/Wh4PU=
|
||||
github.com/shirou/gopsutil/v4 v4.26.1 h1:TOkEyriIXk2HX9d4isZJtbjXbEjf5qyKPAzbzY0JWSo=
|
||||
github.com/shirou/gopsutil/v4 v4.26.1/go.mod h1:medLI9/UNAb0dOI9Q3/7yWSqKkj00u+1tgY8nvv41pc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
@@ -1919,8 +1919,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
|
||||
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
|
||||
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
|
||||
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
|
||||
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
@@ -2582,8 +2582,8 @@ google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q
|
||||
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
|
||||
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc=
|
||||
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba h1:B14OtaXuMaCQsl2deSvNkyPKIzq3BjfxQp8d00QyWx4=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:G5IanEx8/PgI9w6CFcYQf7jMtHQhZruvfM1i3qOqk5U=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 h1:ZdyUkS9po3H7G0tuh955QVyyotWvOD4W0aEapeGeUYk=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846/go.mod h1:Fk4kyraUvqD7i5H6S43sj2W98fbZa75lpZz/eUyhfO0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.07"
|
||||
appVersion: "4.08"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.0.407
|
||||
version: 4.0.408
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# SEAWEEDFS - helm chart (2.x+)
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Add the helm repo
|
||||
|
||||
|
||||
@@ -42,11 +42,11 @@ spec:
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ include "seaweedfs.fullname" . }}-admin
|
||||
name: {{ include "seaweedfs.componentName" (list . "admin") }}
|
||||
port:
|
||||
number: {{ .Values.admin.port }}
|
||||
{{- else }}
|
||||
serviceName: {{ include "seaweedfs.fullname" . }}-admin
|
||||
serviceName: {{ include "seaweedfs.componentName" (list . "admin") }}
|
||||
servicePort: {{ .Values.admin.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.fullname" . }}-admin
|
||||
name: {{ include "seaweedfs.componentName" (list . "admin") }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
@@ -21,7 +21,7 @@ metadata:
|
||||
{{- toYaml .Values.admin.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceName: {{ include "seaweedfs.fullname" . }}-admin
|
||||
serviceName: {{ include "seaweedfs.componentName" (list . "admin") }}
|
||||
podManagementPolicy: {{ .Values.admin.podManagementPolicy }}
|
||||
replicas: {{ .Values.admin.replicas }}
|
||||
selector:
|
||||
|
||||
@@ -57,7 +57,7 @@ spec:
|
||||
priorityClassName: {{ .Values.cosi.priorityClassName | quote }}
|
||||
{{- end }}
|
||||
enableServiceLinks: false
|
||||
serviceAccountName: {{ .Values.global.serviceAccountName }}-objectstorage-provisioner
|
||||
serviceAccountName: {{ include "seaweedfs.componentName" (list . "objectstorage-provisioner") }}
|
||||
{{- if .Values.cosi.initContainers }}
|
||||
initContainers:
|
||||
{{ tpl .Values.cosi.initContainers . | nindent 8 | trim }}
|
||||
@@ -78,16 +78,16 @@ spec:
|
||||
{{- else if .Values.s3.ingress.enabled }}
|
||||
value: "{{ printf "https://%s" .Values.s3.ingress.host }}"
|
||||
{{- else if .Values.s3.enabled }}
|
||||
value: "{{ printf "https://%s-s3.%s.svc" (include "seaweedfs.fullname" .) .Release.Namespace }}"
|
||||
value: "{{ printf "https://%s.%s.svc" (include "seaweedfs.componentName" (list . "s3")) .Release.Namespace }}"
|
||||
{{- else }}
|
||||
value: "{{ printf "https://%s-filer.%s.svc" (include "seaweedfs.fullname" .) .Release.Namespace }}"
|
||||
value: "{{ printf "https://%s.%s.svc" (include "seaweedfs.componentName" (list . "filer")) .Release.Namespace }}"
|
||||
{{- end }}
|
||||
{{- with .Values.cosi.region }}
|
||||
- name: REGION
|
||||
value: "{{ . }}"
|
||||
{{- end }}
|
||||
- name: SEAWEEDFS_FILER
|
||||
value: "{{ include "seaweedfs.fullname" . }}-filer:{{ .Values.filer.grpcPort }}"
|
||||
value: "{{ include "seaweedfs.componentName" (list . "filer") }}:{{ .Values.filer.grpcPort }}"
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
- name: WEED_GRPC_CLIENT_KEY
|
||||
value: /usr/local/share/ca-certificates/client/tls.key
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
{{- $filerEnabled := or .Values.filer.enabled .Values.allInOne.enabled }}
|
||||
{{- if and $filerEnabled .Values.filer.ingress.enabled }}
|
||||
{{- /* Determine service name based on deployment mode */}}
|
||||
{{- $serviceName := ternary (printf "%s-all-in-one" (include "seaweedfs.fullname" .)) (printf "%s-filer" (include "seaweedfs.fullname" .)) .Values.allInOne.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "filer")) .Values.allInOne.enabled }}
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.fullname" . }}-filer
|
||||
name: {{ include "seaweedfs.componentName" (list . "filer") }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
@@ -15,7 +15,7 @@ metadata:
|
||||
{{- toYaml .Values.filer.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceName: {{ include "seaweedfs.fullname" . }}-filer
|
||||
serviceName: {{ include "seaweedfs.componentName" (list . "filer") }}
|
||||
podManagementPolicy: {{ .Values.filer.podManagementPolicy }}
|
||||
replicas: {{ .Values.filer.replicas }}
|
||||
{{- if (gt (int .Values.filer.updatePartition) 0) }}
|
||||
|
||||
@@ -33,15 +33,15 @@ spec:
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ include "seaweedfs.fullname" . }}-master
|
||||
name: {{ include "seaweedfs.componentName" (list . "master") }}
|
||||
port:
|
||||
number: {{ .Values.master.port }}
|
||||
#name:
|
||||
{{- else }}
|
||||
serviceName: {{ include "seaweedfs.fullname" . }}-master
|
||||
serviceName: {{ include "seaweedfs.componentName" (list . "master") }}
|
||||
servicePort: {{ .Values.master.port }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.ingress.host }}
|
||||
{{- if .Values.master.ingress.host }}
|
||||
host: {{ .Values.master.ingress.host }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -9,6 +9,7 @@ metadata:
|
||||
app.kubernetes.io/component: master
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
annotations:
|
||||
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
|
||||
{{- if .Values.master.annotations }}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.fullname" . }}-master
|
||||
name: {{ include "seaweedfs.componentName" (list . "master") }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
@@ -15,7 +15,7 @@ metadata:
|
||||
{{- toYaml .Values.master.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceName: {{ include "seaweedfs.fullname" . }}-master
|
||||
serviceName: {{ include "seaweedfs.componentName" (list . "master") }}
|
||||
podManagementPolicy: {{ .Values.master.podManagementPolicy }}
|
||||
replicas: {{ .Values.master.replicas }}
|
||||
{{- if (gt (int .Values.master.updatePartition) 0) }}
|
||||
@@ -183,7 +183,7 @@ spec:
|
||||
{{- if .Values.master.garbageThreshold }}
|
||||
-garbageThreshold={{ .Values.master.garbageThreshold }} \
|
||||
{{- end }}
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master.{{ .Release.Namespace }} \
|
||||
-ip=${POD_NAME}.{{ include "seaweedfs.componentName" (list . "master") }}.{{ .Release.Namespace }} \
|
||||
-peers={{ include "seaweedfs.masterServers" . }} \
|
||||
{{- range .Values.master.extraArgs }}
|
||||
{{ . }} \
|
||||
|
||||
@@ -149,7 +149,7 @@ spec:
|
||||
{{- if .Values.s3.auditLogConfig }}
|
||||
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
-filer={{ include "seaweedfs.fullname" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }} \
|
||||
-filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ .Release.Namespace }}:{{ .Values.filer.port }} \
|
||||
{{- range .Values.s3.extraArgs }}
|
||||
{{ . }} \
|
||||
{{- end }}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
{{- $s3Enabled := or .Values.s3.enabled (and .Values.filer.s3.enabled (not .Values.allInOne.enabled)) (and .Values.allInOne.enabled .Values.allInOne.s3.enabled) }}
|
||||
{{- if and $s3Enabled .Values.s3.ingress.enabled }}
|
||||
{{- /* Determine service name based on deployment mode */}}
|
||||
{{- $serviceName := ternary (printf "%s-all-in-one" (include "seaweedfs.fullname" .)) (printf "%s-s3" (include "seaweedfs.fullname" .)) .Values.allInOne.enabled }}
|
||||
{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "s3")) .Values.allInOne.enabled }}
|
||||
{{- $s3Port := .Values.allInOne.s3.port | default .Values.s3.port }}
|
||||
{{- /* Build hosts list - support both legacy .host (string) and new .hosts (array) for backwards compatibility */}}
|
||||
{{- $hosts := list }}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
{{- if or (and (or .Values.s3.enabled .Values.allInOne.enabled) .Values.s3.enableAuth (not .Values.s3.existingConfigSecret)) (and .Values.filer.s3.enabled .Values.filer.s3.enableAuth (not .Values.filer.s3.existingConfigSecret)) }}
|
||||
{{- $secretName := printf "%s-s3-secret" (include "seaweedfs.fullname" .) }}
|
||||
{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace (default $secretName .Values.s3.legacySecretName) }}
|
||||
{{- $legacySecretName := "seaweedfs-s3-secret" }}
|
||||
{{- $lookupName := $secretName }}
|
||||
{{- if .Values.s3.reuseLegacySecret }}
|
||||
{{- $lookupName = default $legacySecretName .Values.s3.legacySecretName }}
|
||||
{{- end }}
|
||||
{{- $existingSecret := lookup "v1" "Secret" .Release.Namespace $lookupName }}
|
||||
{{- $reuse := false }}
|
||||
{{- if and .Values.s3.reuseLegacySecret $existingSecret }}
|
||||
{{- $reuse = true }}
|
||||
|
||||
@@ -9,6 +9,7 @@ metadata:
|
||||
app.kubernetes.io/component: s3
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- if .Values.s3.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.s3.annotations | nindent 4 }}
|
||||
|
||||
@@ -171,7 +171,7 @@ spec:
|
||||
-key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{- end }}
|
||||
-userStoreFile=/etc/sw/seaweedfs_sftp_config \
|
||||
-filer={{ include "seaweedfs.fullname" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
|
||||
-filer={{ include "seaweedfs.componentName" (list . "filer-client") }}.{{ .Release.Namespace }}:{{ .Values.filer.port }}
|
||||
volumeMounts:
|
||||
{{- if or (eq .Values.sftp.logs.type "hostPath") (eq .Values.sftp.logs.type "emptyDir") }}
|
||||
- name: logs
|
||||
|
||||
@@ -17,6 +17,22 @@ be used as a full name.
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a truncated component name.
|
||||
Usage: {{ include "seaweedfs.componentName" (list . "component-suffix") }}
|
||||
*/}}
|
||||
{{- define "seaweedfs.componentName" -}}
|
||||
{{- $context := index . 0 -}}
|
||||
{{- $suffix := index . 1 -}}
|
||||
{{- if gt (len $suffix) 61 -}}
|
||||
{{- fail (printf "Suffix '%s' is too long for componentName helper. Max length is 61." $suffix) -}}
|
||||
{{- end -}}
|
||||
{{- $fullname := include "seaweedfs.fullname" $context -}}
|
||||
{{- $maxLen := sub 62 (len $suffix) | int -}}
|
||||
{{- $truncatedFullname := trunc $maxLen $fullname | trimSuffix "-" -}}
|
||||
{{- printf "%s-%s" $truncatedFullname $suffix -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
@@ -261,7 +277,7 @@ If allInOne is enabled, point to the all-in-one service; otherwise, point to the
|
||||
{{- if .Values.allInOne.enabled -}}
|
||||
{{- $serviceNameSuffix = "-all-in-one" -}}
|
||||
{{- end -}}
|
||||
{{- printf "%s%s.%s:%d" (include "seaweedfs.fullname" .) $serviceNameSuffix .Release.Namespace (int .Values.master.port) -}}
|
||||
{{- printf "%s.%s:%d" (printf "%s%s" (include "seaweedfs.fullname" .) $serviceNameSuffix | trunc 63 | trimSuffix "-") .Release.Namespace (int .Values.master.port) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
@@ -273,19 +289,19 @@ If allInOne is enabled, point to the all-in-one service; otherwise, point to the
|
||||
{{- if .Values.allInOne.enabled -}}
|
||||
{{- $serviceNameSuffix = "-all-in-one" -}}
|
||||
{{- end -}}
|
||||
{{- printf "%s%s.%s:%d" (include "seaweedfs.fullname" .) $serviceNameSuffix .Release.Namespace (int .Values.filer.port) -}}
|
||||
{{- printf "%s.%s:%d" (printf "%s%s" (include "seaweedfs.fullname" .) $serviceNameSuffix | trunc 63 | trimSuffix "-") .Release.Namespace (int .Values.filer.port) -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Generate comma-separated list of master server addresses.
|
||||
Usage: {{ include "seaweedfs.masterServers" . }}
|
||||
Output example: ${SEAWEEDFS_FULLNAME}-master-0.${SEAWEEDFS_FULLNAME}-master.namespace:9333,${SEAWEEDFS_FULLNAME}-master-1...
|
||||
Output example: my-release-master-0.my-release-master.namespace:9333,my-release-master-1...
|
||||
*/}}
|
||||
{{- define "seaweedfs.masterServers" -}}
|
||||
{{- $fullname := include "seaweedfs.fullname" . -}}
|
||||
{{- $masterName := include "seaweedfs.componentName" (list . "master") -}}
|
||||
{{- range $index := until (.Values.master.replicas | int) -}}
|
||||
{{- if $index }},{{ end -}}
|
||||
${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}
|
||||
{{ $masterName }}-{{ $index }}.{{ $masterName }}.{{ $.Release.Namespace }}:{{ $.Values.master.port }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
|
||||
@@ -15,14 +15,18 @@
|
||||
{{- $existingConfigSecret = or .Values.allInOne.s3.existingConfigSecret .Values.s3.existingConfigSecret .Values.filer.s3.existingConfigSecret }}
|
||||
{{- end }}
|
||||
{{- else if .Values.master.enabled }}
|
||||
{{- /* Check standalone filer.s3 mode */}}
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
{{- /* Check if embedded (in filer) or standalone S3 gateway is enabled */}}
|
||||
{{- if or .Values.filer.s3.enabled .Values.s3.enabled }}
|
||||
{{- $s3Enabled = true }}
|
||||
{{- if .Values.filer.s3.createBuckets }}
|
||||
{{- if .Values.s3.createBuckets }}
|
||||
{{- $createBuckets = .Values.s3.createBuckets }}
|
||||
{{- $enableAuth = .Values.s3.enableAuth }}
|
||||
{{- $existingConfigSecret = .Values.s3.existingConfigSecret }}
|
||||
{{- else if .Values.filer.s3.createBuckets }}
|
||||
{{- $createBuckets = .Values.filer.s3.createBuckets }}
|
||||
{{- $enableAuth = .Values.filer.s3.enableAuth }}
|
||||
{{- $existingConfigSecret = .Values.filer.s3.existingConfigSecret }}
|
||||
{{- end }}
|
||||
{{- $enableAuth = .Values.filer.s3.enableAuth }}
|
||||
{{- $existingConfigSecret = .Values.filer.s3.existingConfigSecret }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -36,7 +40,7 @@ metadata:
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name | quote }}
|
||||
annotations:
|
||||
"helm.sh/hook": post-install
|
||||
"helm.sh/hook": post-install,post-upgrade
|
||||
"helm.sh/hook-weight": "-5"
|
||||
"helm.sh/hook-delete-policy": hook-succeeded
|
||||
spec:
|
||||
@@ -105,9 +109,12 @@ spec:
|
||||
wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}"
|
||||
{{- end }}
|
||||
{{- range $createBuckets }}
|
||||
/bin/echo \
|
||||
"s3.bucket.create --name {{ .name }}" |\
|
||||
/usr/bin/weed shell
|
||||
if /bin/echo "s3.bucket.list" | /usr/bin/weed shell | awk '{print $1}' | grep -Fxq "{{ .name }}"; then
|
||||
echo "Bucket '{{ .name }}' already exists, skipping creation."
|
||||
else
|
||||
echo "Creating bucket '{{ .name }}'..."
|
||||
/bin/echo "s3.bucket.create --name {{ .name }}" | /usr/bin/weed shell
|
||||
fi
|
||||
{{- end }}
|
||||
{{- range $createBuckets }}
|
||||
{{- if .anonymousRead }}
|
||||
|
||||
@@ -15,7 +15,6 @@ metadata:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
stringData:
|
||||
user: "YourSWUser"
|
||||
password: "HardCodedPassword"
|
||||
# better to random generate and create in DB
|
||||
# password: {{ randAlphaNum 10 | sha256sum | b64enc | trunc 32 }}
|
||||
# auto-generated password
|
||||
password: {{ randAlphaNum 10 | sha256sum | b64enc | trunc 32 }}
|
||||
{{- end }}
|
||||
|
||||
@@ -48,7 +48,7 @@ data:
|
||||
{{- if .Values.global.securityConfig.jwtSigning.filerRead }}
|
||||
# If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
|
||||
# - f.e. the S3 API Shim generates the JWT
|
||||
# - the Filer server validates the JWT on writing
|
||||
# - the Filer server validates the JWT on reading
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
[jwt.filer_signing.read]
|
||||
key = "{{ dig "jwt" "filer_signing" "read" "key" (randAlphaNum 10 | b64enc) $securityConfig }}"
|
||||
|
||||
@@ -42,11 +42,11 @@ spec:
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ include "seaweedfs.fullname" . }}-volume
|
||||
name: {{ include "seaweedfs.componentName" (list . "volume") }}
|
||||
port:
|
||||
number: {{ .Values.volume.port }}
|
||||
{{- else }}
|
||||
serviceName: {{ include "seaweedfs.fullname" . }}-volume
|
||||
serviceName: {{ include "seaweedfs.componentName" (list . "volume") }}
|
||||
servicePort: {{ .Values.volume.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.fullname" $ }}-{{ $volumeName }}
|
||||
name: {{ include "seaweedfs.componentName" (list $ $volumeName) }}
|
||||
namespace: {{ $.Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" $ }}
|
||||
@@ -21,7 +21,7 @@ metadata:
|
||||
{{- toYaml $volume.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
serviceName: {{ include "seaweedfs.fullname" $ }}-{{ $volumeName }}
|
||||
serviceName: {{ include "seaweedfs.componentName" (list $ $volumeName) }}
|
||||
replicas: {{ $volume.replicas }}
|
||||
podManagementPolicy: {{ $volume.podManagementPolicy }}
|
||||
selector:
|
||||
@@ -197,7 +197,7 @@ spec:
|
||||
-fileSizeLimitMB={{ $volume.fileSizeLimitMB }} \
|
||||
{{- end }}
|
||||
-minFreeSpacePercent={{ $volume.minFreeSpacePercent }} \
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-{{ $volumeName }}.{{ $.Release.Namespace }} \
|
||||
-ip=${POD_NAME}.{{ include "seaweedfs.componentName" (list $ $volumeName) }}.{{ $.Release.Namespace }} \
|
||||
-compactionMBps={{ $volume.compactionMBps }} \
|
||||
-master={{ include "seaweedfs.masterServerArg" $ }} \
|
||||
{{- range $volume.extraArgs }}
|
||||
|
||||
@@ -891,7 +891,7 @@ filer:
|
||||
# should have a secret key called seaweedfs_s3_config with an inline json configure
|
||||
existingConfigSecret: null
|
||||
auditLogConfig: {}
|
||||
# You may specify buckets to be created during the install process.
|
||||
# You may specify buckets to be created during the install or upgrade process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
# createBuckets:
|
||||
# - name: bucket-a
|
||||
@@ -916,6 +916,13 @@ s3:
|
||||
# should have a secret key called seaweedfs_s3_config with an inline json config
|
||||
existingConfigSecret: null
|
||||
auditLogConfig: {}
|
||||
# You may specify buckets to be created during the install or upgrade process.
|
||||
# Buckets may be exposed publicly by setting `anonymousRead` to `true`
|
||||
# createBuckets:
|
||||
# - name: bucket-a
|
||||
# anonymousRead: true
|
||||
# - name: bucket-b
|
||||
# anonymousRead: false
|
||||
|
||||
# Suffix of the host name, {bucket}.{domainName}
|
||||
domainName: ""
|
||||
|
||||
@@ -129,6 +129,7 @@ func TestS3IAMDistributedTests(t *testing.T) {
|
||||
errorMsg := err.Error()
|
||||
return strings.Contains(errorMsg, "timeout") ||
|
||||
strings.Contains(errorMsg, "connection reset") ||
|
||||
strings.Contains(errorMsg, "connection refused") ||
|
||||
strings.Contains(errorMsg, "temporary failure") ||
|
||||
strings.Contains(errorMsg, "TooManyRequests") ||
|
||||
strings.Contains(errorMsg, "ServiceUnavailable") ||
|
||||
|
||||
@@ -0,0 +1,431 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
func getFirstNamespace(namespace []string) (string, error) {
|
||||
if len(namespace) == 0 {
|
||||
return "", fmt.Errorf("namespace must not be empty")
|
||||
}
|
||||
return namespace[0], nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) doRestRequest(method, path string, body interface{}) (*http.Response, error) {
|
||||
var bodyBytes []byte
|
||||
var err error
|
||||
|
||||
if body != nil {
|
||||
bodyBytes, err = json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request body: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(method, c.endpoint+path, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
|
||||
}
|
||||
|
||||
if err := c.signRequest(req, bodyBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return c.client.Do(req)
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) doTargetRequest(operation string, body interface{}) (*http.Response, error) {
|
||||
var bodyBytes []byte
|
||||
var err error
|
||||
|
||||
if body != nil {
|
||||
bodyBytes, err = json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal request body: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(http.MethodPost, c.endpoint+"/", bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.URL.RawPath = "/"
|
||||
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
|
||||
req.Header.Set("X-Amz-Target", "S3Tables."+operation)
|
||||
|
||||
if err := c.signRequest(req, bodyBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return c.client.Do(req)
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) doTargetRequestAndDecode(operation string, reqBody interface{}, respBody interface{}) error {
|
||||
resp, err := c.doTargetRequest(operation, reqBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("%s failed with status %d and could not read error response body: %v", operation, resp.StatusCode, readErr)
|
||||
}
|
||||
var errResp s3tables.S3TablesError
|
||||
if err := json.Unmarshal(bodyBytes, &errResp); err != nil {
|
||||
return fmt.Errorf("%s failed with status %d, could not decode error response: %v. Body: %s", operation, resp.StatusCode, err, string(bodyBytes))
|
||||
}
|
||||
return fmt.Errorf("%s failed: %s - %s", operation, errResp.Type, errResp.Message)
|
||||
}
|
||||
|
||||
if respBody != nil {
|
||||
if err := json.NewDecoder(resp.Body).Decode(respBody); err != nil {
|
||||
return fmt.Errorf("failed to decode %s response: %w", operation, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) signRequest(req *http.Request, body []byte) error {
|
||||
creds := aws.Credentials{
|
||||
AccessKeyID: c.accessKey,
|
||||
SecretAccessKey: c.secretKey,
|
||||
}
|
||||
if req.Host == "" {
|
||||
req.Host = req.URL.Host
|
||||
}
|
||||
req.Header.Set("Host", req.URL.Host)
|
||||
payloadHash := sha256.Sum256(body)
|
||||
return v4.NewSigner().SignHTTP(context.Background(), creds, req, hex.EncodeToString(payloadHash[:]), "s3tables", c.region, time.Now())
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) doRestRequestAndDecode(operation, method, path string, reqBody interface{}, respBody interface{}) error {
|
||||
resp, err := c.doRestRequest(method, path, reqBody)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bodyBytes, readErr := io.ReadAll(resp.Body)
|
||||
if readErr != nil {
|
||||
return fmt.Errorf("%s failed with status %d and could not read error response body: %v", operation, resp.StatusCode, readErr)
|
||||
}
|
||||
var errResp s3tables.S3TablesError
|
||||
if err := json.Unmarshal(bodyBytes, &errResp); err != nil {
|
||||
return fmt.Errorf("%s failed with status %d, could not decode error response: %v. Body: %s", operation, resp.StatusCode, err, string(bodyBytes))
|
||||
}
|
||||
return fmt.Errorf("%s failed: %s - %s", operation, errResp.Type, errResp.Message)
|
||||
}
|
||||
|
||||
if respBody != nil {
|
||||
if err := json.NewDecoder(resp.Body).Decode(respBody); err != nil {
|
||||
return fmt.Errorf("failed to decode %s response: %w", operation, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Table Bucket operations
|
||||
|
||||
func (c *S3TablesClient) CreateTableBucket(name string, tags map[string]string) (*s3tables.CreateTableBucketResponse, error) {
|
||||
req := &s3tables.CreateTableBucketRequest{
|
||||
Name: name,
|
||||
Tags: tags,
|
||||
}
|
||||
var result s3tables.CreateTableBucketResponse
|
||||
if err := c.doRestRequestAndDecode("CreateTableBucket", http.MethodPut, "/buckets", req, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) GetTableBucket(arn string) (*s3tables.GetTableBucketResponse, error) {
|
||||
path := "/buckets/" + url.PathEscape(arn)
|
||||
var result s3tables.GetTableBucketResponse
|
||||
if err := c.doRestRequestAndDecode("GetTableBucket", http.MethodGet, path, nil, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) ListTableBuckets(prefix, continuationToken string, maxBuckets int) (*s3tables.ListTableBucketsResponse, error) {
|
||||
query := url.Values{}
|
||||
if prefix != "" {
|
||||
query.Set("prefix", prefix)
|
||||
}
|
||||
if continuationToken != "" {
|
||||
query.Set("continuationToken", continuationToken)
|
||||
}
|
||||
if maxBuckets > 0 {
|
||||
query.Set("maxBuckets", strconv.Itoa(maxBuckets))
|
||||
}
|
||||
path := "/buckets"
|
||||
if encoded := query.Encode(); encoded != "" {
|
||||
path = path + "?" + encoded
|
||||
}
|
||||
var result s3tables.ListTableBucketsResponse
|
||||
if err := c.doRestRequestAndDecode("ListTableBuckets", http.MethodGet, path, nil, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) DeleteTableBucket(arn string) error {
|
||||
path := "/buckets/" + url.PathEscape(arn)
|
||||
return c.doRestRequestAndDecode("DeleteTableBucket", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
// Namespace operations
|
||||
|
||||
func (c *S3TablesClient) CreateNamespace(bucketARN string, namespace []string) (*s3tables.CreateNamespaceResponse, error) {
|
||||
if len(namespace) == 0 {
|
||||
return nil, fmt.Errorf("CreateNamespace requires namespace")
|
||||
}
|
||||
req := &s3tables.CreateNamespaceRequest{
|
||||
Namespace: namespace,
|
||||
}
|
||||
path := "/namespaces/" + url.PathEscape(bucketARN)
|
||||
var result s3tables.CreateNamespaceResponse
|
||||
if err := c.doRestRequestAndDecode("CreateNamespace", http.MethodPut, path, req, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) GetNamespace(bucketARN string, namespace []string) (*s3tables.GetNamespaceResponse, error) {
|
||||
name, err := getFirstNamespace(namespace)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetNamespace requires namespace: %w", err)
|
||||
}
|
||||
path := "/namespaces/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(name)
|
||||
var result s3tables.GetNamespaceResponse
|
||||
if err := c.doRestRequestAndDecode("GetNamespace", http.MethodGet, path, nil, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) ListNamespaces(bucketARN, prefix, continuationToken string, maxNamespaces int) (*s3tables.ListNamespacesResponse, error) {
|
||||
query := url.Values{}
|
||||
if prefix != "" {
|
||||
query.Set("prefix", prefix)
|
||||
}
|
||||
if continuationToken != "" {
|
||||
query.Set("continuationToken", continuationToken)
|
||||
}
|
||||
if maxNamespaces > 0 {
|
||||
query.Set("maxNamespaces", strconv.Itoa(maxNamespaces))
|
||||
}
|
||||
path := "/namespaces/" + url.PathEscape(bucketARN)
|
||||
if encoded := query.Encode(); encoded != "" {
|
||||
path = path + "?" + encoded
|
||||
}
|
||||
var result s3tables.ListNamespacesResponse
|
||||
if err := c.doRestRequestAndDecode("ListNamespaces", http.MethodGet, path, nil, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) DeleteNamespace(bucketARN string, namespace []string) error {
|
||||
name, err := getFirstNamespace(namespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("DeleteNamespace requires namespace: %w", err)
|
||||
}
|
||||
path := "/namespaces/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(name)
|
||||
return c.doRestRequestAndDecode("DeleteNamespace", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
// Table operations
|
||||
|
||||
func (c *S3TablesClient) CreateTable(bucketARN string, namespace []string, name, format string, metadata *s3tables.TableMetadata, tags map[string]string) (*s3tables.CreateTableResponse, error) {
|
||||
nameSpace, err := getFirstNamespace(namespace)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("CreateTable requires namespace: %w", err)
|
||||
}
|
||||
req := &s3tables.CreateTableRequest{
|
||||
Name: name,
|
||||
Format: format,
|
||||
Metadata: metadata,
|
||||
Tags: tags,
|
||||
}
|
||||
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(nameSpace)
|
||||
var result s3tables.CreateTableResponse
|
||||
if err := c.doRestRequestAndDecode("CreateTable", http.MethodPut, path, req, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) GetTable(bucketARN string, namespace []string, name string) (*s3tables.GetTableResponse, error) {
|
||||
nameSpace, err := getFirstNamespace(namespace)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetTable requires namespace: %w", err)
|
||||
}
|
||||
query := url.Values{}
|
||||
query.Set("tableBucketARN", bucketARN)
|
||||
query.Set("namespace", nameSpace)
|
||||
query.Set("name", name)
|
||||
path := "/get-table?" + query.Encode()
|
||||
var result s3tables.GetTableResponse
|
||||
if err := c.doRestRequestAndDecode("GetTable", http.MethodGet, path, nil, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) ListTables(bucketARN string, namespace []string, prefix, continuationToken string, maxTables int) (*s3tables.ListTablesResponse, error) {
|
||||
query := url.Values{}
|
||||
if len(namespace) > 0 {
|
||||
nameSpace, err := getFirstNamespace(namespace)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ListTables requires namespace: %w", err)
|
||||
}
|
||||
query.Set("namespace", nameSpace)
|
||||
}
|
||||
if prefix != "" {
|
||||
query.Set("prefix", prefix)
|
||||
}
|
||||
if continuationToken != "" {
|
||||
query.Set("continuationToken", continuationToken)
|
||||
}
|
||||
if maxTables > 0 {
|
||||
query.Set("maxTables", strconv.Itoa(maxTables))
|
||||
}
|
||||
path := "/tables/" + url.PathEscape(bucketARN)
|
||||
if encoded := query.Encode(); encoded != "" {
|
||||
path = path + "?" + encoded
|
||||
}
|
||||
var result s3tables.ListTablesResponse
|
||||
if err := c.doRestRequestAndDecode("ListTables", http.MethodGet, path, nil, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) DeleteTable(bucketARN string, namespace []string, name string) error {
|
||||
nameSpace, err := getFirstNamespace(namespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("DeleteTable requires namespace: %w", err)
|
||||
}
|
||||
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(nameSpace) + "/" + url.PathEscape(name)
|
||||
return c.doRestRequestAndDecode("DeleteTable", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
// Policy operations
|
||||
|
||||
func (c *S3TablesClient) PutTableBucketPolicy(bucketARN, policy string) error {
|
||||
req := &s3tables.PutTableBucketPolicyRequest{
|
||||
ResourcePolicy: policy,
|
||||
}
|
||||
path := "/buckets/" + url.PathEscape(bucketARN) + "/policy"
|
||||
return c.doRestRequestAndDecode("PutTableBucketPolicy", http.MethodPut, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) GetTableBucketPolicy(bucketARN string) (*s3tables.GetTableBucketPolicyResponse, error) {
|
||||
path := "/buckets/" + url.PathEscape(bucketARN) + "/policy"
|
||||
var result s3tables.GetTableBucketPolicyResponse
|
||||
if err := c.doRestRequestAndDecode("GetTableBucketPolicy", http.MethodGet, path, nil, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) DeleteTableBucketPolicy(bucketARN string) error {
|
||||
path := "/buckets/" + url.PathEscape(bucketARN) + "/policy"
|
||||
return c.doRestRequestAndDecode("DeleteTableBucketPolicy", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
// Table Policy operations
|
||||
|
||||
func (c *S3TablesClient) PutTablePolicy(bucketARN string, namespace []string, name, policy string) error {
|
||||
nameSpace, err := getFirstNamespace(namespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("PutTablePolicy requires namespace: %w", err)
|
||||
}
|
||||
req := &s3tables.PutTablePolicyRequest{
|
||||
ResourcePolicy: policy,
|
||||
}
|
||||
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(nameSpace) + "/" + url.PathEscape(name) + "/policy"
|
||||
return c.doRestRequestAndDecode("PutTablePolicy", http.MethodPut, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) GetTablePolicy(bucketARN string, namespace []string, name string) (*s3tables.GetTablePolicyResponse, error) {
|
||||
nameSpace, err := getFirstNamespace(namespace)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetTablePolicy requires namespace: %w", err)
|
||||
}
|
||||
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(nameSpace) + "/" + url.PathEscape(name) + "/policy"
|
||||
var result s3tables.GetTablePolicyResponse
|
||||
if err := c.doRestRequestAndDecode("GetTablePolicy", http.MethodGet, path, nil, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) DeleteTablePolicy(bucketARN string, namespace []string, name string) error {
|
||||
nameSpace, err := getFirstNamespace(namespace)
|
||||
if err != nil {
|
||||
return fmt.Errorf("DeleteTablePolicy requires namespace: %w", err)
|
||||
}
|
||||
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(nameSpace) + "/" + url.PathEscape(name) + "/policy"
|
||||
return c.doRestRequestAndDecode("DeleteTablePolicy", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
|
||||
// Tagging operations
|
||||
|
||||
func (c *S3TablesClient) TagResource(resourceARN string, tags map[string]string) error {
|
||||
req := &s3tables.TagResourceRequest{
|
||||
Tags: tags,
|
||||
}
|
||||
path := "/tag/" + url.PathEscape(resourceARN)
|
||||
return c.doRestRequestAndDecode("TagResource", http.MethodPost, path, req, nil)
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) ListTagsForResource(resourceARN string) (*s3tables.ListTagsForResourceResponse, error) {
|
||||
path := "/tag/" + url.PathEscape(resourceARN)
|
||||
var result s3tables.ListTagsForResourceResponse
|
||||
if err := c.doRestRequestAndDecode("ListTagsForResource", http.MethodGet, path, nil, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *S3TablesClient) UntagResource(resourceARN string, tagKeys []string) error {
|
||||
if len(tagKeys) == 0 {
|
||||
return fmt.Errorf("tagKeys cannot be empty")
|
||||
}
|
||||
query := url.Values{}
|
||||
for _, key := range tagKeys {
|
||||
query.Add("tagKeys", key)
|
||||
}
|
||||
path := "/tag/" + url.PathEscape(resourceARN)
|
||||
if encoded := query.Encode(); encoded != "" {
|
||||
path = path + "?" + encoded
|
||||
}
|
||||
return c.doRestRequestAndDecode("UntagResource", http.MethodDelete, path, nil, nil)
|
||||
}
|
||||
@@ -0,0 +1,699 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cryptorand "crypto/rand"
|
||||
"sync"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/command"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
|
||||
)
|
||||
|
||||
var (
|
||||
miniClusterMutex sync.Mutex
|
||||
)
|
||||
|
||||
func TestS3TablesIntegration(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
// Create and start test cluster
|
||||
cluster, err := startMiniCluster(t)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
// Create S3 Tables client
|
||||
client := NewS3TablesClient(cluster.s3Endpoint, testRegion, testAccessKey, testSecretKey)
|
||||
|
||||
// Run test suite
|
||||
t.Run("TableBucketLifecycle", func(t *testing.T) {
|
||||
testTableBucketLifecycle(t, client)
|
||||
})
|
||||
|
||||
t.Run("NamespaceLifecycle", func(t *testing.T) {
|
||||
testNamespaceLifecycle(t, client)
|
||||
})
|
||||
|
||||
t.Run("TableLifecycle", func(t *testing.T) {
|
||||
testTableLifecycle(t, client)
|
||||
})
|
||||
|
||||
t.Run("TableBucketPolicy", func(t *testing.T) {
|
||||
testTableBucketPolicy(t, client)
|
||||
})
|
||||
|
||||
t.Run("TablePolicy", func(t *testing.T) {
|
||||
testTablePolicy(t, client)
|
||||
})
|
||||
|
||||
t.Run("Tagging", func(t *testing.T) {
|
||||
testTagging(t, client)
|
||||
})
|
||||
|
||||
t.Run("TargetOperations", func(t *testing.T) {
|
||||
testTargetOperations(t, client)
|
||||
})
|
||||
}
|
||||
|
||||
func testTableBucketLifecycle(t *testing.T, client *S3TablesClient) {
|
||||
bucketName := "test-bucket-" + randomString(8)
|
||||
|
||||
// Create table bucket
|
||||
createResp, err := client.CreateTableBucket(bucketName, nil)
|
||||
require.NoError(t, err, "Failed to create table bucket")
|
||||
assert.Contains(t, createResp.ARN, bucketName)
|
||||
t.Logf("✓ Created table bucket: %s", createResp.ARN)
|
||||
|
||||
// Get table bucket
|
||||
getResp, err := client.GetTableBucket(createResp.ARN)
|
||||
require.NoError(t, err, "Failed to get table bucket")
|
||||
assert.Equal(t, bucketName, getResp.Name)
|
||||
t.Logf("✓ Got table bucket: %s", getResp.Name)
|
||||
|
||||
// List table buckets
|
||||
listResp, err := client.ListTableBuckets("", "", 0)
|
||||
require.NoError(t, err, "Failed to list table buckets")
|
||||
found := false
|
||||
for _, b := range listResp.TableBuckets {
|
||||
if b.Name == bucketName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Created bucket should appear in list")
|
||||
t.Logf("✓ Listed table buckets, found %d buckets", len(listResp.TableBuckets))
|
||||
|
||||
// Delete table bucket
|
||||
err = client.DeleteTableBucket(createResp.ARN)
|
||||
require.NoError(t, err, "Failed to delete table bucket")
|
||||
t.Logf("✓ Deleted table bucket: %s", bucketName)
|
||||
|
||||
// Verify bucket is deleted
|
||||
_, err = client.GetTableBucket(createResp.ARN)
|
||||
assert.Error(t, err, "Bucket should not exist after deletion")
|
||||
}
|
||||
|
||||
func testNamespaceLifecycle(t *testing.T, client *S3TablesClient) {
|
||||
bucketName := "test-ns-bucket-" + randomString(8)
|
||||
namespaceName := "test_namespace"
|
||||
|
||||
// Create table bucket first
|
||||
createBucketResp, err := client.CreateTableBucket(bucketName, nil)
|
||||
require.NoError(t, err, "Failed to create table bucket")
|
||||
defer client.DeleteTableBucket(createBucketResp.ARN)
|
||||
|
||||
bucketARN := createBucketResp.ARN
|
||||
|
||||
// Create namespace
|
||||
createNsResp, err := client.CreateNamespace(bucketARN, []string{namespaceName})
|
||||
require.NoError(t, err, "Failed to create namespace")
|
||||
assert.Equal(t, []string{namespaceName}, createNsResp.Namespace)
|
||||
t.Logf("✓ Created namespace: %s", namespaceName)
|
||||
|
||||
// Get namespace
|
||||
getNsResp, err := client.GetNamespace(bucketARN, []string{namespaceName})
|
||||
require.NoError(t, err, "Failed to get namespace")
|
||||
assert.Equal(t, []string{namespaceName}, getNsResp.Namespace)
|
||||
t.Logf("✓ Got namespace: %v", getNsResp.Namespace)
|
||||
|
||||
// List namespaces
|
||||
listNsResp, err := client.ListNamespaces(bucketARN, "", "", 0)
|
||||
require.NoError(t, err, "Failed to list namespaces")
|
||||
found := false
|
||||
for _, ns := range listNsResp.Namespaces {
|
||||
if len(ns.Namespace) > 0 && ns.Namespace[0] == namespaceName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Created namespace should appear in list")
|
||||
t.Logf("✓ Listed namespaces, found %d namespaces", len(listNsResp.Namespaces))
|
||||
|
||||
// Delete namespace
|
||||
err = client.DeleteNamespace(bucketARN, []string{namespaceName})
|
||||
require.NoError(t, err, "Failed to delete namespace")
|
||||
t.Logf("✓ Deleted namespace: %s", namespaceName)
|
||||
|
||||
// Verify namespace is deleted
|
||||
_, err = client.GetNamespace(bucketARN, []string{namespaceName})
|
||||
assert.Error(t, err, "Namespace should not exist after deletion")
|
||||
}
|
||||
|
||||
func testTableLifecycle(t *testing.T, client *S3TablesClient) {
|
||||
bucketName := "test-table-bucket-" + randomString(8)
|
||||
namespaceName := "test_ns"
|
||||
tableName := "test_table"
|
||||
|
||||
// Create table bucket
|
||||
createBucketResp, err := client.CreateTableBucket(bucketName, nil)
|
||||
require.NoError(t, err, "Failed to create table bucket")
|
||||
defer client.DeleteTableBucket(createBucketResp.ARN)
|
||||
|
||||
bucketARN := createBucketResp.ARN
|
||||
|
||||
// Create namespace
|
||||
_, err = client.CreateNamespace(bucketARN, []string{namespaceName})
|
||||
require.NoError(t, err, "Failed to create namespace")
|
||||
defer client.DeleteNamespace(bucketARN, []string{namespaceName})
|
||||
|
||||
// Create table with Iceberg schema
|
||||
icebergMetadata := &s3tables.TableMetadata{
|
||||
Iceberg: &s3tables.IcebergMetadata{
|
||||
Schema: s3tables.IcebergSchema{
|
||||
Fields: []s3tables.IcebergSchemaField{
|
||||
{Name: "id", Type: "int", Required: true},
|
||||
{Name: "name", Type: "string"},
|
||||
{Name: "value", Type: "int"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
createTableResp, err := client.CreateTable(bucketARN, []string{namespaceName}, tableName, "ICEBERG", icebergMetadata, nil)
|
||||
require.NoError(t, err, "Failed to create table")
|
||||
assert.NotEmpty(t, createTableResp.TableARN)
|
||||
assert.NotEmpty(t, createTableResp.VersionToken)
|
||||
t.Logf("✓ Created table: %s (version: %s)", createTableResp.TableARN, createTableResp.VersionToken)
|
||||
|
||||
// Get table
|
||||
getTableResp, err := client.GetTable(bucketARN, []string{namespaceName}, tableName)
|
||||
require.NoError(t, err, "Failed to get table")
|
||||
assert.Equal(t, tableName, getTableResp.Name)
|
||||
assert.Equal(t, "ICEBERG", getTableResp.Format)
|
||||
t.Logf("✓ Got table: %s (format: %s)", getTableResp.Name, getTableResp.Format)
|
||||
|
||||
// List tables
|
||||
listTablesResp, err := client.ListTables(bucketARN, []string{namespaceName}, "", "", 0)
|
||||
require.NoError(t, err, "Failed to list tables")
|
||||
found := false
|
||||
for _, tbl := range listTablesResp.Tables {
|
||||
if tbl.Name == tableName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Created table should appear in list")
|
||||
t.Logf("✓ Listed tables, found %d tables", len(listTablesResp.Tables))
|
||||
|
||||
// Delete table
|
||||
err = client.DeleteTable(bucketARN, []string{namespaceName}, tableName)
|
||||
require.NoError(t, err, "Failed to delete table")
|
||||
t.Logf("✓ Deleted table: %s", tableName)
|
||||
|
||||
// Verify table is deleted
|
||||
_, err = client.GetTable(bucketARN, []string{namespaceName}, tableName)
|
||||
assert.Error(t, err, "Table should not exist after deletion")
|
||||
}
|
||||
|
||||
func testTableBucketPolicy(t *testing.T, client *S3TablesClient) {
|
||||
bucketName := "test-policy-bucket-" + randomString(8)
|
||||
|
||||
// Create table bucket
|
||||
createBucketResp, err := client.CreateTableBucket(bucketName, nil)
|
||||
require.NoError(t, err, "Failed to create table bucket")
|
||||
defer client.DeleteTableBucket(createBucketResp.ARN)
|
||||
|
||||
bucketARN := createBucketResp.ARN
|
||||
|
||||
// Put bucket policy
|
||||
policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3tables:*","Resource":"*"}]}`
|
||||
err = client.PutTableBucketPolicy(bucketARN, policy)
|
||||
require.NoError(t, err, "Failed to put table bucket policy")
|
||||
t.Logf("✓ Put table bucket policy")
|
||||
|
||||
// Get bucket policy
|
||||
getPolicyResp, err := client.GetTableBucketPolicy(bucketARN)
|
||||
require.NoError(t, err, "Failed to get table bucket policy")
|
||||
assert.Equal(t, policy, getPolicyResp.ResourcePolicy)
|
||||
t.Logf("✓ Got table bucket policy")
|
||||
|
||||
// Delete bucket policy
|
||||
err = client.DeleteTableBucketPolicy(bucketARN)
|
||||
require.NoError(t, err, "Failed to delete table bucket policy")
|
||||
t.Logf("✓ Deleted table bucket policy")
|
||||
|
||||
// Verify policy is deleted
|
||||
_, err = client.GetTableBucketPolicy(bucketARN)
|
||||
assert.Error(t, err, "Policy should not exist after deletion")
|
||||
}
|
||||
|
||||
func testTablePolicy(t *testing.T, client *S3TablesClient) {
|
||||
bucketName := "test-table-policy-bucket-" + randomString(8)
|
||||
namespaceName := "test_ns"
|
||||
tableName := "test_table"
|
||||
|
||||
// Create table bucket
|
||||
createBucketResp, err := client.CreateTableBucket(bucketName, nil)
|
||||
require.NoError(t, err, "Failed to create table bucket")
|
||||
defer client.DeleteTableBucket(createBucketResp.ARN)
|
||||
|
||||
bucketARN := createBucketResp.ARN
|
||||
|
||||
// Create namespace
|
||||
_, err = client.CreateNamespace(bucketARN, []string{namespaceName})
|
||||
require.NoError(t, err, "Failed to create namespace")
|
||||
defer client.DeleteNamespace(bucketARN, []string{namespaceName})
|
||||
|
||||
// Create table
|
||||
icebergMetadata := &s3tables.TableMetadata{
|
||||
Iceberg: &s3tables.IcebergMetadata{
|
||||
Schema: s3tables.IcebergSchema{
|
||||
Fields: []s3tables.IcebergSchemaField{
|
||||
{Name: "id", Type: "int", Required: true},
|
||||
{Name: "name", Type: "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
createTableResp, err := client.CreateTable(bucketARN, []string{namespaceName}, tableName, "ICEBERG", icebergMetadata, nil)
|
||||
require.NoError(t, err, "Failed to create table")
|
||||
defer client.DeleteTable(bucketARN, []string{namespaceName}, tableName)
|
||||
|
||||
t.Logf("✓ Created table: %s", createTableResp.TableARN)
|
||||
|
||||
// Verify no policy exists initially
|
||||
_, err = client.GetTablePolicy(bucketARN, []string{namespaceName}, tableName)
|
||||
assert.Error(t, err, "Policy should not exist initially")
|
||||
t.Logf("✓ Verified no policy exists initially")
|
||||
|
||||
// Put table policy
|
||||
policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3tables:*","Resource":"*"}]}`
|
||||
err = client.PutTablePolicy(bucketARN, []string{namespaceName}, tableName, policy)
|
||||
require.NoError(t, err, "Failed to put table policy")
|
||||
t.Logf("✓ Put table policy")
|
||||
|
||||
// Get table policy
|
||||
getPolicyResp, err := client.GetTablePolicy(bucketARN, []string{namespaceName}, tableName)
|
||||
require.NoError(t, err, "Failed to get table policy")
|
||||
assert.Equal(t, policy, getPolicyResp.ResourcePolicy)
|
||||
t.Logf("✓ Got table policy")
|
||||
|
||||
// Delete table policy
|
||||
err = client.DeleteTablePolicy(bucketARN, []string{namespaceName}, tableName)
|
||||
require.NoError(t, err, "Failed to delete table policy")
|
||||
t.Logf("✓ Deleted table policy")
|
||||
|
||||
// Verify policy is deleted
|
||||
_, err = client.GetTablePolicy(bucketARN, []string{namespaceName}, tableName)
|
||||
assert.Error(t, err, "Policy should not exist after deletion")
|
||||
t.Logf("✓ Verified policy deletion")
|
||||
}
|
||||
|
||||
func testTagging(t *testing.T, client *S3TablesClient) {
|
||||
bucketName := "test-tag-bucket-" + randomString(8)
|
||||
|
||||
// Create table bucket with tags
|
||||
initialTags := map[string]string{"Environment": "test"}
|
||||
createBucketResp, err := client.CreateTableBucket(bucketName, initialTags)
|
||||
require.NoError(t, err, "Failed to create table bucket")
|
||||
defer client.DeleteTableBucket(createBucketResp.ARN)
|
||||
|
||||
bucketARN := createBucketResp.ARN
|
||||
|
||||
// List tags
|
||||
listTagsResp, err := client.ListTagsForResource(bucketARN)
|
||||
require.NoError(t, err, "Failed to list tags")
|
||||
assert.Equal(t, "test", listTagsResp.Tags["Environment"])
|
||||
t.Logf("✓ Listed tags: %v", listTagsResp.Tags)
|
||||
|
||||
// Add more tags
|
||||
newTags := map[string]string{"Department": "Engineering"}
|
||||
err = client.TagResource(bucketARN, newTags)
|
||||
require.NoError(t, err, "Failed to tag resource")
|
||||
t.Logf("✓ Added tags")
|
||||
|
||||
// Verify tags
|
||||
listTagsResp, err = client.ListTagsForResource(bucketARN)
|
||||
require.NoError(t, err, "Failed to list tags")
|
||||
assert.Equal(t, "test", listTagsResp.Tags["Environment"])
|
||||
assert.Equal(t, "Engineering", listTagsResp.Tags["Department"])
|
||||
t.Logf("✓ Verified tags: %v", listTagsResp.Tags)
|
||||
|
||||
// Remove a tag
|
||||
err = client.UntagResource(bucketARN, []string{"Environment"})
|
||||
require.NoError(t, err, "Failed to untag resource")
|
||||
t.Logf("✓ Removed tag")
|
||||
|
||||
// Verify tag is removed
|
||||
listTagsResp, err = client.ListTagsForResource(bucketARN)
|
||||
require.NoError(t, err, "Failed to list tags")
|
||||
_, hasEnvironment := listTagsResp.Tags["Environment"]
|
||||
assert.False(t, hasEnvironment, "Environment tag should be removed")
|
||||
assert.Equal(t, "Engineering", listTagsResp.Tags["Department"])
|
||||
t.Logf("✓ Verified tag removal")
|
||||
}
|
||||
|
||||
func testTargetOperations(t *testing.T, client *S3TablesClient) {
|
||||
bucketName := "test-target-bucket-" + randomString(8)
|
||||
|
||||
var createResp s3tables.CreateTableBucketResponse
|
||||
err := client.doTargetRequestAndDecode("CreateTableBucket", &s3tables.CreateTableBucketRequest{
|
||||
Name: bucketName,
|
||||
}, &createResp)
|
||||
require.NoError(t, err, "Failed to create table bucket via target")
|
||||
defer client.doTargetRequestAndDecode("DeleteTableBucket", &s3tables.DeleteTableBucketRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
}, nil)
|
||||
|
||||
var listResp s3tables.ListTableBucketsResponse
|
||||
err = client.doTargetRequestAndDecode("ListTableBuckets", &s3tables.ListTableBucketsRequest{}, &listResp)
|
||||
require.NoError(t, err, "Failed to list table buckets via target")
|
||||
found := false
|
||||
for _, b := range listResp.TableBuckets {
|
||||
if b.Name == bucketName {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Created bucket should appear in target list")
|
||||
|
||||
var getResp s3tables.GetTableBucketResponse
|
||||
err = client.doTargetRequestAndDecode("GetTableBucket", &s3tables.GetTableBucketRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
}, &getResp)
|
||||
require.NoError(t, err, "Failed to get table bucket via target")
|
||||
assert.Equal(t, bucketName, getResp.Name)
|
||||
|
||||
namespaceName := "target_ns"
|
||||
var createNsResp s3tables.CreateNamespaceResponse
|
||||
err = client.doTargetRequestAndDecode("CreateNamespace", &s3tables.CreateNamespaceRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
Namespace: []string{namespaceName},
|
||||
}, &createNsResp)
|
||||
require.NoError(t, err, "Failed to create namespace via target")
|
||||
defer client.doTargetRequestAndDecode("DeleteNamespace", &s3tables.DeleteNamespaceRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
Namespace: []string{namespaceName},
|
||||
}, nil)
|
||||
|
||||
var listNsResp s3tables.ListNamespacesResponse
|
||||
err = client.doTargetRequestAndDecode("ListNamespaces", &s3tables.ListNamespacesRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
}, &listNsResp)
|
||||
require.NoError(t, err, "Failed to list namespaces via target")
|
||||
|
||||
tableName := "target_table"
|
||||
var createTableResp s3tables.CreateTableResponse
|
||||
err = client.doTargetRequestAndDecode("CreateTable", &s3tables.CreateTableRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
Namespace: []string{namespaceName},
|
||||
Name: tableName,
|
||||
Format: "ICEBERG",
|
||||
}, &createTableResp)
|
||||
require.NoError(t, err, "Failed to create table via target")
|
||||
defer client.doTargetRequestAndDecode("DeleteTable", &s3tables.DeleteTableRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
Namespace: []string{namespaceName},
|
||||
Name: tableName,
|
||||
}, nil)
|
||||
|
||||
var listTablesResp s3tables.ListTablesResponse
|
||||
err = client.doTargetRequestAndDecode("ListTables", &s3tables.ListTablesRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
Namespace: []string{namespaceName},
|
||||
}, &listTablesResp)
|
||||
require.NoError(t, err, "Failed to list tables via target")
|
||||
|
||||
var getTableResp s3tables.GetTableResponse
|
||||
err = client.doTargetRequestAndDecode("GetTable", &s3tables.GetTableRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
Namespace: []string{namespaceName},
|
||||
Name: tableName,
|
||||
}, &getTableResp)
|
||||
require.NoError(t, err, "Failed to get table via target")
|
||||
assert.Equal(t, tableName, getTableResp.Name)
|
||||
|
||||
policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3tables:*","Resource":"*"}]}`
|
||||
err = client.doTargetRequestAndDecode("PutTableBucketPolicy", &s3tables.PutTableBucketPolicyRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
ResourcePolicy: policy,
|
||||
}, nil)
|
||||
require.NoError(t, err, "Failed to put bucket policy via target")
|
||||
|
||||
var getPolicyResp s3tables.GetTableBucketPolicyResponse
|
||||
err = client.doTargetRequestAndDecode("GetTableBucketPolicy", &s3tables.GetTableBucketPolicyRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
}, &getPolicyResp)
|
||||
require.NoError(t, err, "Failed to get bucket policy via target")
|
||||
assert.Equal(t, policy, getPolicyResp.ResourcePolicy)
|
||||
|
||||
err = client.doTargetRequestAndDecode("DeleteTableBucketPolicy", &s3tables.DeleteTableBucketPolicyRequest{
|
||||
TableBucketARN: createResp.ARN,
|
||||
}, nil)
|
||||
require.NoError(t, err, "Failed to delete bucket policy via target")
|
||||
|
||||
err = client.doTargetRequestAndDecode("TagResource", &s3tables.TagResourceRequest{
|
||||
ResourceARN: createResp.ARN,
|
||||
Tags: map[string]string{"Environment": "test"},
|
||||
}, nil)
|
||||
require.NoError(t, err, "Failed to tag resource via target")
|
||||
|
||||
var listTagsResp s3tables.ListTagsForResourceResponse
|
||||
err = client.doTargetRequestAndDecode("ListTagsForResource", &s3tables.ListTagsForResourceRequest{
|
||||
ResourceARN: createResp.ARN,
|
||||
}, &listTagsResp)
|
||||
require.NoError(t, err, "Failed to list tags via target")
|
||||
assert.Equal(t, "test", listTagsResp.Tags["Environment"])
|
||||
|
||||
err = client.doTargetRequestAndDecode("UntagResource", &s3tables.UntagResourceRequest{
|
||||
ResourceARN: createResp.ARN,
|
||||
TagKeys: []string{"Environment"},
|
||||
}, nil)
|
||||
require.NoError(t, err, "Failed to untag resource via target")
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
|
||||
// findAvailablePort finds an available port by binding to port 0
|
||||
func findAvailablePort() (int, error) {
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
addr := listener.Addr().(*net.TCPAddr)
|
||||
return addr.Port, nil
|
||||
}
|
||||
|
||||
// startMiniCluster starts a weed mini instance directly without exec
|
||||
func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
// Find available ports
|
||||
masterPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find master port: %v", err)
|
||||
}
|
||||
masterGrpcPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find master grpc port: %v", err)
|
||||
}
|
||||
volumePort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find volume port: %v", err)
|
||||
}
|
||||
volumeGrpcPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find volume grpc port: %v", err)
|
||||
}
|
||||
filerPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find filer port: %v", err)
|
||||
}
|
||||
filerGrpcPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find filer grpc port: %v", err)
|
||||
}
|
||||
s3Port, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find s3 port: %v", err)
|
||||
}
|
||||
s3GrpcPort, err := findAvailablePort()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find s3 grpc port: %v", err)
|
||||
}
|
||||
// Create temporary directory for test data
|
||||
testDir := t.TempDir()
|
||||
|
||||
// Ensure no configuration file from previous runs
|
||||
configFile := filepath.Join(testDir, "mini.options")
|
||||
_ = os.Remove(configFile)
|
||||
|
||||
// Create context with timeout
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
s3Endpoint := fmt.Sprintf("http://127.0.0.1:%d", s3Port)
|
||||
cluster := &TestCluster{
|
||||
t: t,
|
||||
dataDir: testDir,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
masterPort: masterPort,
|
||||
volumePort: volumePort,
|
||||
filerPort: filerPort,
|
||||
s3Port: s3Port,
|
||||
s3Endpoint: s3Endpoint,
|
||||
}
|
||||
|
||||
// Create empty security.toml to disable JWT authentication in tests
|
||||
securityToml := filepath.Join(testDir, "security.toml")
|
||||
err = os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("failed to create security.toml: %v", err)
|
||||
}
|
||||
|
||||
// Start weed mini in a goroutine by calling the command directly
|
||||
cluster.wg.Add(1)
|
||||
go func() {
|
||||
defer cluster.wg.Done()
|
||||
|
||||
// Protect global state mutation with a mutex
|
||||
miniClusterMutex.Lock()
|
||||
defer miniClusterMutex.Unlock()
|
||||
|
||||
// Save current directory and args
|
||||
oldDir, _ := os.Getwd()
|
||||
oldArgs := os.Args
|
||||
defer func() {
|
||||
os.Chdir(oldDir)
|
||||
os.Args = oldArgs
|
||||
}()
|
||||
|
||||
// Change to test directory so mini picks up security.toml
|
||||
os.Chdir(testDir)
|
||||
|
||||
// Configure args for mini command
|
||||
os.Args = []string{
|
||||
"weed",
|
||||
"-dir=" + testDir,
|
||||
"-master.port=" + strconv.Itoa(masterPort),
|
||||
"-master.port.grpc=" + strconv.Itoa(masterGrpcPort),
|
||||
"-volume.port=" + strconv.Itoa(volumePort),
|
||||
"-volume.port.grpc=" + strconv.Itoa(volumeGrpcPort),
|
||||
"-filer.port=" + strconv.Itoa(filerPort),
|
||||
"-filer.port.grpc=" + strconv.Itoa(filerGrpcPort),
|
||||
"-s3.port=" + strconv.Itoa(s3Port),
|
||||
"-s3.port.grpc=" + strconv.Itoa(s3GrpcPort),
|
||||
"-webdav.port=0", // Disable WebDAV
|
||||
"-admin.ui=false", // Disable admin UI
|
||||
"-master.volumeSizeLimitMB=32", // Small volumes for testing
|
||||
"-ip=127.0.0.1",
|
||||
"-master.peers=none", // Faster startup
|
||||
"-s3.iam.readOnly=false", // Enable IAM write operations for tests
|
||||
}
|
||||
|
||||
// Suppress most logging during tests
|
||||
glog.MaxSize = 1024 * 1024
|
||||
|
||||
// Find and run the mini command
|
||||
for _, cmd := range command.Commands {
|
||||
if cmd.Name() == "mini" && cmd.Run != nil {
|
||||
cmd.Flag.Parse(os.Args[1:])
|
||||
args := cmd.Flag.Args()
|
||||
command.MiniClusterCtx = ctx
|
||||
cmd.Run(cmd, args)
|
||||
command.MiniClusterCtx = nil
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for S3 service to be ready
|
||||
err = waitForS3Ready(cluster.s3Endpoint, 30*time.Second)
|
||||
if err != nil {
|
||||
cancel()
|
||||
return nil, fmt.Errorf("S3 service failed to start: %v", err)
|
||||
}
|
||||
|
||||
cluster.isRunning = true
|
||||
|
||||
t.Logf("Test cluster started successfully at %s", cluster.s3Endpoint)
|
||||
return cluster, nil
|
||||
}
|
||||
|
||||
// Stop stops the test cluster
|
||||
func (c *TestCluster) Stop() {
|
||||
if c.cancel != nil {
|
||||
c.cancel()
|
||||
}
|
||||
// Give services time to shut down gracefully
|
||||
if c.isRunning {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
// Wait for the mini goroutine to finish
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
c.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
timer := time.NewTimer(2 * time.Second)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-done:
|
||||
// Goroutine finished
|
||||
case <-timer.C:
|
||||
// Timeout - goroutine doesn't respond to context cancel
|
||||
// This may indicate the mini cluster didn't shut down cleanly
|
||||
c.t.Log("Warning: Test cluster shutdown timed out after 2 seconds")
|
||||
}
|
||||
|
||||
// Reset the global cmdMini flags to prevent state leakage to other tests
|
||||
for _, cmd := range command.Commands {
|
||||
if cmd.Name() == "mini" {
|
||||
// Reset flags to defaults
|
||||
cmd.Flag.VisitAll(func(f *flag.Flag) {
|
||||
// Reset to default value
|
||||
f.Value.Set(f.DefValue)
|
||||
})
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitForS3Ready waits for the S3 service to be ready
|
||||
func waitForS3Ready(endpoint string, timeout time.Duration) error {
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
deadline := time.Now().Add(timeout)
|
||||
|
||||
for time.Now().Before(deadline) {
|
||||
resp, err := client.Get(endpoint)
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
// Wait a bit more to ensure service is fully ready
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
return nil
|
||||
}
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
|
||||
return fmt.Errorf("timeout waiting for S3 service at %s", endpoint)
|
||||
}
|
||||
|
||||
// randomString generates a random string for unique naming
|
||||
func randomString(length int) string {
|
||||
const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
b := make([]byte, length)
|
||||
if _, err := cryptorand.Read(b); err != nil {
|
||||
panic("failed to generate random string: " + err.Error())
|
||||
}
|
||||
for i := range b {
|
||||
b[i] = charset[int(b[i])%len(charset)]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestCluster manages the weed mini instance for integration testing
|
||||
type TestCluster struct {
|
||||
t *testing.T
|
||||
dataDir string
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
isRunning bool
|
||||
startOnce sync.Once
|
||||
wg sync.WaitGroup
|
||||
masterPort int
|
||||
volumePort int
|
||||
filerPort int
|
||||
s3Port int
|
||||
s3Endpoint string
|
||||
}
|
||||
|
||||
// S3TablesClient is a simple client for S3 Tables API
|
||||
type S3TablesClient struct {
|
||||
endpoint string
|
||||
region string
|
||||
accessKey string
|
||||
secretKey string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewS3TablesClient creates a new S3 Tables client
|
||||
func NewS3TablesClient(endpoint, region, accessKey, secretKey string) *S3TablesClient {
|
||||
return &S3TablesClient{
|
||||
endpoint: endpoint,
|
||||
region: region,
|
||||
accessKey: accessKey,
|
||||
secretKey: secretKey,
|
||||
client: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// Test configuration constants
|
||||
const (
|
||||
testRegion = "us-west-2"
|
||||
testAccessKey = "admin"
|
||||
testSecretKey = "admin"
|
||||
testAccountID = "111122223333"
|
||||
)
|
||||
@@ -8,6 +8,7 @@ A modern web-based administration interface for SeaweedFS clusters built with Go
|
||||
- **Master Management**: Monitor master nodes and leadership status
|
||||
- **Volume Server Management**: View volume servers, capacity, and health
|
||||
- **Object Store Bucket Management**: Create, delete, and manage Object Store buckets with web interface
|
||||
- **S3 Tables Management**: Manage table buckets, namespaces, tables, tags, and policies via the admin UI
|
||||
- **System Health**: Overall cluster health monitoring
|
||||
- **Responsive Design**: Bootstrap-based UI that works on all devices
|
||||
- **Authentication**: Optional user authentication with sessions
|
||||
@@ -96,7 +97,6 @@ make fmt
|
||||
weed/admin/
|
||||
├── Makefile # Admin-specific build tasks
|
||||
├── README.md # This file
|
||||
├── S3_BUCKETS.md # Object Store bucket management documentation
|
||||
├── admin.go # Main application entry point
|
||||
├── dash/ # Server and handler logic
|
||||
│ ├── admin_server.go # HTTP server setup
|
||||
@@ -110,20 +110,20 @@ weed/admin/
|
||||
├── app/ # Application templates
|
||||
│ ├── admin.templ # Main dashboard template
|
||||
│ ├── s3_buckets.templ # Object Store bucket management template
|
||||
│ ├── s3tables_*.templ # S3 Tables management templates
|
||||
│ └── *_templ.go # Generated Go code
|
||||
└── layout/ # Layout templates
|
||||
├── layout.templ # Base layout template
|
||||
└── layout_templ.go # Generated Go code
|
||||
```
|
||||
|
||||
### S3 Bucket Management
|
||||
### Object Store Management
|
||||
|
||||
The admin interface includes comprehensive Object Store bucket management capabilities. See [S3_BUCKETS.md](S3_BUCKETS.md) for detailed documentation on:
|
||||
The admin interface includes Object Store and S3 Tables management capabilities:
|
||||
|
||||
- Creating and deleting Object Store buckets
|
||||
- Viewing bucket contents and metadata
|
||||
- Managing bucket permissions and settings
|
||||
- API endpoints for programmatic access
|
||||
- Create/delete Object Store buckets and adjust quotas or ownership.
|
||||
- Manage S3 Tables buckets, namespaces, and tables.
|
||||
- Update S3 Tables policies and tags via the UI and API endpoints.
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -276,4 +276,4 @@ The admin component follows a clean architecture:
|
||||
- **Business Logic**: Handler functions in `dash/` package
|
||||
- **Data Layer**: Communicates with SeaweedFS masters and filers
|
||||
|
||||
This separation makes the code maintainable and testable.
|
||||
This separation makes the code maintainable and testable.
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/credential/grpc" // Register gRPC credential store
|
||||
@@ -101,6 +102,8 @@ type AdminServer struct {
|
||||
collectionStatsCache map[string]collectionStats
|
||||
lastCollectionStatsUpdate time.Time
|
||||
collectionStatsCacheThreshold time.Duration
|
||||
|
||||
s3TablesManager *s3tables.Manager
|
||||
}
|
||||
|
||||
// Type definitions moved to types.go
|
||||
@@ -132,6 +135,7 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string)
|
||||
filerCacheExpiration: 30 * time.Second, // Cache filers for 30 seconds
|
||||
configPersistence: NewConfigPersistence(dataDir),
|
||||
collectionStatsCacheThreshold: 30 * time.Second,
|
||||
s3TablesManager: newS3TablesManager(),
|
||||
}
|
||||
|
||||
// Initialize topic retention purger
|
||||
|
||||
@@ -39,9 +39,11 @@ type FileBrowserData struct {
|
||||
Breadcrumbs []BreadcrumbItem `json:"breadcrumbs"`
|
||||
Entries []FileEntry `json:"entries"`
|
||||
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
IsBucketPath bool `json:"is_bucket_path"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
IsBucketPath bool `json:"is_bucket_path"`
|
||||
BucketName string `json:"bucket_name"`
|
||||
IsTableBucketPath bool `json:"is_table_bucket_path"`
|
||||
TableBucketName string `json:"table_bucket_name"`
|
||||
// Pagination fields
|
||||
PageSize int `json:"page_size"`
|
||||
HasNextPage bool `json:"has_next_page"`
|
||||
@@ -227,15 +229,28 @@ func (s *AdminServer) GetFileBrowser(dir string, lastFileName string, pageSize i
|
||||
}
|
||||
}
|
||||
|
||||
// Check if this is a table bucket path
|
||||
isTableBucketPath := false
|
||||
tableBucketName := ""
|
||||
if strings.HasPrefix(dir, "/table-buckets/") {
|
||||
isTableBucketPath = true
|
||||
pathParts := strings.Split(strings.Trim(dir, "/"), "/")
|
||||
if len(pathParts) >= 2 {
|
||||
tableBucketName = pathParts[1]
|
||||
}
|
||||
}
|
||||
|
||||
return &FileBrowserData{
|
||||
CurrentPath: dir,
|
||||
ParentPath: parentPath,
|
||||
Breadcrumbs: breadcrumbs,
|
||||
Entries: entries,
|
||||
|
||||
LastUpdated: time.Now(),
|
||||
IsBucketPath: isBucketPath,
|
||||
BucketName: bucketName,
|
||||
LastUpdated: time.Now(),
|
||||
IsBucketPath: isBucketPath,
|
||||
BucketName: bucketName,
|
||||
IsTableBucketPath: isTableBucketPath,
|
||||
TableBucketName: tableBucketName,
|
||||
// Pagination metadata
|
||||
PageSize: pageSize,
|
||||
HasNextPage: hasNextPage,
|
||||
@@ -272,8 +287,12 @@ func (s *AdminServer) generateBreadcrumbs(dir string) []BreadcrumbItem {
|
||||
displayName := part
|
||||
if len(breadcrumbs) == 1 && part == "buckets" {
|
||||
displayName = "Object Store Buckets"
|
||||
} else if len(breadcrumbs) == 1 && part == "table-buckets" {
|
||||
displayName = "Table Buckets"
|
||||
} else if len(breadcrumbs) == 2 && strings.HasPrefix(dir, "/buckets/") {
|
||||
displayName = "📦 " + part // Add bucket icon to bucket name
|
||||
} else if len(breadcrumbs) == 2 && strings.HasPrefix(dir, "/table-buckets/") {
|
||||
displayName = "🧊 " + part
|
||||
}
|
||||
|
||||
breadcrumbs = append(breadcrumbs, BreadcrumbItem{
|
||||
|
||||
@@ -51,6 +51,15 @@ func TestGenerateBreadcrumbs(t *testing.T) {
|
||||
{Name: "📦 mybucket", Path: "/buckets/mybucket"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "table bucket path",
|
||||
path: "/table-buckets/mytablebucket",
|
||||
expected: []BreadcrumbItem{
|
||||
{Name: "Root", Path: "/"},
|
||||
{Name: "Table Buckets", Path: "/table-buckets"},
|
||||
{Name: "🧊 mytablebucket", Path: "/table-buckets/mytablebucket"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bucket nested path",
|
||||
path: "/buckets/mybucket/folder",
|
||||
@@ -61,6 +70,16 @@ func TestGenerateBreadcrumbs(t *testing.T) {
|
||||
{Name: "folder", Path: "/buckets/mybucket/folder"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "table bucket nested path",
|
||||
path: "/table-buckets/mytablebucket/folder",
|
||||
expected: []BreadcrumbItem{
|
||||
{Name: "Root", Path: "/"},
|
||||
{Name: "Table Buckets", Path: "/table-buckets"},
|
||||
{Name: "🧊 mytablebucket", Path: "/table-buckets/mytablebucket"},
|
||||
{Name: "folder", Path: "/table-buckets/mytablebucket/folder"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "path with trailing slash",
|
||||
path: "/folder/",
|
||||
@@ -176,6 +195,11 @@ func TestParentPathCalculationLogic(t *testing.T) {
|
||||
currentDir: "/buckets/mybucket",
|
||||
expected: "/buckets",
|
||||
},
|
||||
{
|
||||
name: "table bucket directory",
|
||||
currentDir: "/table-buckets/mytablebucket",
|
||||
expected: "/table-buckets",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -0,0 +1,605 @@
|
||||
package dash
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// S3Tables data structures for admin UI
|
||||
|
||||
type S3TablesBucketsData struct {
|
||||
Username string `json:"username"`
|
||||
Buckets []S3TablesBucketSummary `json:"buckets"`
|
||||
TotalBuckets int `json:"total_buckets"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
type S3TablesBucketSummary struct {
|
||||
ARN string `json:"arn"`
|
||||
Name string `json:"name"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type S3TablesNamespacesData struct {
|
||||
Username string `json:"username"`
|
||||
BucketARN string `json:"bucket_arn"`
|
||||
Namespaces []s3tables.NamespaceSummary `json:"namespaces"`
|
||||
TotalNamespaces int `json:"total_namespaces"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
type S3TablesTablesData struct {
|
||||
Username string `json:"username"`
|
||||
BucketARN string `json:"bucket_arn"`
|
||||
Namespace string `json:"namespace"`
|
||||
Tables []s3tables.TableSummary `json:"tables"`
|
||||
TotalTables int `json:"total_tables"`
|
||||
LastUpdated time.Time `json:"last_updated"`
|
||||
}
|
||||
|
||||
type tableBucketMetadata struct {
|
||||
Name string `json:"name"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
OwnerAccountID string `json:"ownerAccountId"`
|
||||
}
|
||||
|
||||
// S3Tables manager helpers
|
||||
|
||||
const s3TablesAdminListLimit = 1000
|
||||
|
||||
func newS3TablesManager() *s3tables.Manager {
|
||||
manager := s3tables.NewManager()
|
||||
manager.SetAccountID(s3_constants.AccountAdminId)
|
||||
return manager
|
||||
}
|
||||
|
||||
func (s *AdminServer) executeS3TablesOperation(ctx context.Context, operation string, req interface{}, resp interface{}) error {
|
||||
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
mgrClient := s3tables.NewManagerClient(client)
|
||||
return s.s3TablesManager.Execute(ctx, mgrClient, operation, req, resp, s3_constants.AccountAdminId)
|
||||
})
|
||||
}
|
||||
|
||||
// S3Tables data retrieval for pages
|
||||
|
||||
func (s *AdminServer) GetS3TablesBucketsData(ctx context.Context) (S3TablesBucketsData, error) {
|
||||
var buckets []S3TablesBucketSummary
|
||||
err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
|
||||
Directory: s3tables.TablesPath,
|
||||
Limit: uint32(s3TablesAdminListLimit * 2),
|
||||
InclusiveStartFrom: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for len(buckets) < s3TablesAdminListLimit {
|
||||
entry, recvErr := resp.Recv()
|
||||
if recvErr != nil {
|
||||
if recvErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return recvErr
|
||||
}
|
||||
if entry.Entry == nil || !entry.Entry.IsDirectory {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
continue
|
||||
}
|
||||
metaBytes, ok := entry.Entry.Extended[s3tables.ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var metadata tableBucketMetadata
|
||||
if err := json.Unmarshal(metaBytes, &metadata); err != nil {
|
||||
glog.V(1).Infof("S3Tables: failed to decode table bucket metadata for %s: %v", entry.Entry.Name, err)
|
||||
continue
|
||||
}
|
||||
arn, err := s3tables.BuildBucketARN(s3tables.DefaultRegion, metadata.OwnerAccountID, entry.Entry.Name)
|
||||
if err != nil {
|
||||
glog.V(1).Infof("S3Tables: failed to build table bucket ARN for %s: %v", entry.Entry.Name, err)
|
||||
continue
|
||||
}
|
||||
buckets = append(buckets, S3TablesBucketSummary{
|
||||
ARN: arn,
|
||||
Name: entry.Entry.Name,
|
||||
OwnerAccountID: metadata.OwnerAccountID,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return S3TablesBucketsData{}, err
|
||||
}
|
||||
return S3TablesBucketsData{
|
||||
Buckets: buckets,
|
||||
TotalBuckets: len(buckets),
|
||||
LastUpdated: time.Now(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetS3TablesNamespacesData(ctx context.Context, bucketArn string) (S3TablesNamespacesData, error) {
|
||||
var resp s3tables.ListNamespacesResponse
|
||||
req := &s3tables.ListNamespacesRequest{TableBucketARN: bucketArn, MaxNamespaces: s3TablesAdminListLimit}
|
||||
if err := s.executeS3TablesOperation(ctx, "ListNamespaces", req, &resp); err != nil {
|
||||
return S3TablesNamespacesData{}, err
|
||||
}
|
||||
return S3TablesNamespacesData{
|
||||
BucketARN: bucketArn,
|
||||
Namespaces: resp.Namespaces,
|
||||
TotalNamespaces: len(resp.Namespaces),
|
||||
LastUpdated: time.Now(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetS3TablesTablesData(ctx context.Context, bucketArn, namespace string) (S3TablesTablesData, error) {
|
||||
var resp s3tables.ListTablesResponse
|
||||
var ns []string
|
||||
if namespace != "" {
|
||||
ns = []string{namespace}
|
||||
}
|
||||
req := &s3tables.ListTablesRequest{TableBucketARN: bucketArn, Namespace: ns, MaxTables: s3TablesAdminListLimit}
|
||||
if err := s.executeS3TablesOperation(ctx, "ListTables", req, &resp); err != nil {
|
||||
return S3TablesTablesData{}, err
|
||||
}
|
||||
return S3TablesTablesData{
|
||||
BucketARN: bucketArn,
|
||||
Namespace: namespace,
|
||||
Tables: resp.Tables,
|
||||
TotalTables: len(resp.Tables),
|
||||
LastUpdated: time.Now(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// API handlers
|
||||
|
||||
func (s *AdminServer) ListS3TablesBucketsAPI(c *gin.Context) {
|
||||
data, err := s.GetS3TablesBucketsData(c.Request.Context())
|
||||
if err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, data)
|
||||
}
|
||||
|
||||
func (s *AdminServer) CreateS3TablesBucket(c *gin.Context) {
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
Tags map[string]string `json:"tags"`
|
||||
Owner string `json:"owner"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if req.Name == "" {
|
||||
c.JSON(400, gin.H{"error": "Bucket name is required"})
|
||||
return
|
||||
}
|
||||
owner := strings.TrimSpace(req.Owner)
|
||||
if len(owner) > MaxOwnerNameLength {
|
||||
c.JSON(400, gin.H{"error": fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength)})
|
||||
return
|
||||
}
|
||||
if len(req.Tags) > 0 {
|
||||
if err := s3tables.ValidateTags(req.Tags); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid tags: " + err.Error()})
|
||||
return
|
||||
}
|
||||
}
|
||||
createReq := &s3tables.CreateTableBucketRequest{Name: req.Name, Tags: req.Tags}
|
||||
var resp s3tables.CreateTableBucketResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "CreateTableBucket", createReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
if owner != "" {
|
||||
if err := s.SetTableBucketOwner(c.Request.Context(), req.Name, owner); err != nil {
|
||||
deleteReq := &s3tables.DeleteTableBucketRequest{TableBucketARN: resp.ARN}
|
||||
if deleteErr := s.executeS3TablesOperation(c.Request.Context(), "DeleteTableBucket", deleteReq, nil); deleteErr != nil {
|
||||
c.JSON(500, gin.H{"error": fmt.Sprintf("Failed to set table bucket owner: %v; rollback delete failed: %v", err, deleteErr)})
|
||||
return
|
||||
}
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
c.JSON(201, gin.H{"arn": resp.ARN})
|
||||
}
|
||||
|
||||
func (s *AdminServer) SetTableBucketOwner(ctx context.Context, bucketName, owner string) error {
|
||||
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.LookupDirectoryEntry(ctx, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: s3tables.TablesPath,
|
||||
Name: bucketName,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("lookup table bucket %s: %w", bucketName, err)
|
||||
}
|
||||
if resp.Entry == nil {
|
||||
return fmt.Errorf("table bucket %s not found", bucketName)
|
||||
}
|
||||
entry := resp.Entry
|
||||
if entry.Extended == nil {
|
||||
return fmt.Errorf("table bucket %s metadata missing", bucketName)
|
||||
}
|
||||
metaBytes, ok := entry.Extended[s3tables.ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
return fmt.Errorf("table bucket %s metadata missing", bucketName)
|
||||
}
|
||||
var metadata tableBucketMetadata
|
||||
if err := json.Unmarshal(metaBytes, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to parse table bucket metadata: %w", err)
|
||||
}
|
||||
metadata.OwnerAccountID = owner
|
||||
updated, err := json.Marshal(&metadata)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal table bucket metadata: %w", err)
|
||||
}
|
||||
entry.Extended[s3tables.ExtendedKeyMetadata] = updated
|
||||
if _, err := client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
|
||||
Directory: s3tables.TablesPath,
|
||||
Entry: entry,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("failed to update table bucket owner: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3TablesBucket(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
if bucketArn == "" {
|
||||
c.JSON(400, gin.H{"error": "Bucket ARN is required"})
|
||||
return
|
||||
}
|
||||
req := &s3tables.DeleteTableBucketRequest{TableBucketARN: bucketArn}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteTableBucket", req, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Bucket deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) ListS3TablesNamespacesAPI(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
if bucketArn == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket query parameter is required"})
|
||||
return
|
||||
}
|
||||
data, err := s.GetS3TablesNamespacesData(c.Request.Context(), bucketArn)
|
||||
if err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, data)
|
||||
}
|
||||
|
||||
func (s *AdminServer) CreateS3TablesNamespace(c *gin.Context) {
|
||||
var req struct {
|
||||
BucketARN string `json:"bucket_arn"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if req.BucketARN == "" || req.Name == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket_arn and name are required"})
|
||||
return
|
||||
}
|
||||
createReq := &s3tables.CreateNamespaceRequest{TableBucketARN: req.BucketARN, Namespace: []string{req.Name}}
|
||||
var resp s3tables.CreateNamespaceResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "CreateNamespace", createReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(201, gin.H{"namespace": resp.Namespace})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3TablesNamespace(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
namespace := c.Query("name")
|
||||
if bucketArn == "" || namespace == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket and name query parameters are required"})
|
||||
return
|
||||
}
|
||||
req := &s3tables.DeleteNamespaceRequest{TableBucketARN: bucketArn, Namespace: []string{namespace}}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteNamespace", req, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Namespace deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) ListS3TablesTablesAPI(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
if bucketArn == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket query parameter is required"})
|
||||
return
|
||||
}
|
||||
namespace := c.Query("namespace")
|
||||
data, err := s.GetS3TablesTablesData(c.Request.Context(), bucketArn, namespace)
|
||||
if err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, data)
|
||||
}
|
||||
|
||||
func (s *AdminServer) CreateS3TablesTable(c *gin.Context) {
|
||||
var req struct {
|
||||
BucketARN string `json:"bucket_arn"`
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Format string `json:"format"`
|
||||
Tags map[string]string `json:"tags"`
|
||||
Metadata *s3tables.TableMetadata `json:"metadata"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if req.BucketARN == "" || req.Namespace == "" || req.Name == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket_arn, namespace, and name are required"})
|
||||
return
|
||||
}
|
||||
format := req.Format
|
||||
if format == "" {
|
||||
format = "ICEBERG"
|
||||
}
|
||||
if len(req.Tags) > 0 {
|
||||
if err := s3tables.ValidateTags(req.Tags); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid tags: " + err.Error()})
|
||||
return
|
||||
}
|
||||
}
|
||||
createReq := &s3tables.CreateTableRequest{
|
||||
TableBucketARN: req.BucketARN,
|
||||
Namespace: []string{req.Namespace},
|
||||
Name: req.Name,
|
||||
Format: format,
|
||||
Tags: req.Tags,
|
||||
Metadata: req.Metadata,
|
||||
}
|
||||
var resp s3tables.CreateTableResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "CreateTable", createReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(201, gin.H{"table_arn": resp.TableARN, "version_token": resp.VersionToken})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3TablesTable(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
namespace := c.Query("namespace")
|
||||
name := c.Query("name")
|
||||
version := c.Query("version")
|
||||
if bucketArn == "" || namespace == "" || name == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket, namespace, and name query parameters are required"})
|
||||
return
|
||||
}
|
||||
req := &s3tables.DeleteTableRequest{TableBucketARN: bucketArn, Namespace: []string{namespace}, Name: name, VersionToken: version}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteTable", req, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Table deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) PutS3TablesBucketPolicy(c *gin.Context) {
|
||||
var req struct {
|
||||
BucketARN string `json:"bucket_arn"`
|
||||
Policy string `json:"policy"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if req.BucketARN == "" || req.Policy == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket_arn and policy are required"})
|
||||
return
|
||||
}
|
||||
putReq := &s3tables.PutTableBucketPolicyRequest{TableBucketARN: req.BucketARN, ResourcePolicy: req.Policy}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "PutTableBucketPolicy", putReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Policy updated"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetS3TablesBucketPolicy(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
if bucketArn == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket query parameter is required"})
|
||||
return
|
||||
}
|
||||
getReq := &s3tables.GetTableBucketPolicyRequest{TableBucketARN: bucketArn}
|
||||
var resp s3tables.GetTableBucketPolicyResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "GetTableBucketPolicy", getReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"policy": resp.ResourcePolicy})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3TablesBucketPolicy(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
if bucketArn == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket query parameter is required"})
|
||||
return
|
||||
}
|
||||
deleteReq := &s3tables.DeleteTableBucketPolicyRequest{TableBucketARN: bucketArn}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteTableBucketPolicy", deleteReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Policy deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) PutS3TablesTablePolicy(c *gin.Context) {
|
||||
var req struct {
|
||||
BucketARN string `json:"bucket_arn"`
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Policy string `json:"policy"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if req.BucketARN == "" || req.Namespace == "" || req.Name == "" || req.Policy == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket_arn, namespace, name, and policy are required"})
|
||||
return
|
||||
}
|
||||
putReq := &s3tables.PutTablePolicyRequest{TableBucketARN: req.BucketARN, Namespace: []string{req.Namespace}, Name: req.Name, ResourcePolicy: req.Policy}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "PutTablePolicy", putReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Policy updated"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) GetS3TablesTablePolicy(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
namespace := c.Query("namespace")
|
||||
name := c.Query("name")
|
||||
if bucketArn == "" || namespace == "" || name == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket, namespace, and name query parameters are required"})
|
||||
return
|
||||
}
|
||||
getReq := &s3tables.GetTablePolicyRequest{TableBucketARN: bucketArn, Namespace: []string{namespace}, Name: name}
|
||||
var resp s3tables.GetTablePolicyResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "GetTablePolicy", getReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"policy": resp.ResourcePolicy})
|
||||
}
|
||||
|
||||
func (s *AdminServer) DeleteS3TablesTablePolicy(c *gin.Context) {
|
||||
bucketArn := c.Query("bucket")
|
||||
namespace := c.Query("namespace")
|
||||
name := c.Query("name")
|
||||
if bucketArn == "" || namespace == "" || name == "" {
|
||||
c.JSON(400, gin.H{"error": "bucket, namespace, and name query parameters are required"})
|
||||
return
|
||||
}
|
||||
deleteReq := &s3tables.DeleteTablePolicyRequest{TableBucketARN: bucketArn, Namespace: []string{namespace}, Name: name}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "DeleteTablePolicy", deleteReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Policy deleted"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) TagS3TablesResource(c *gin.Context) {
|
||||
var req struct {
|
||||
ResourceARN string `json:"resource_arn"`
|
||||
Tags map[string]string `json:"tags"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if req.ResourceARN == "" || len(req.Tags) == 0 {
|
||||
c.JSON(400, gin.H{"error": "resource_arn and tags are required"})
|
||||
return
|
||||
}
|
||||
if err := s3tables.ValidateTags(req.Tags); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid tags: " + err.Error()})
|
||||
return
|
||||
}
|
||||
tagReq := &s3tables.TagResourceRequest{ResourceARN: req.ResourceARN, Tags: req.Tags}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "TagResource", tagReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Tags updated"})
|
||||
}
|
||||
|
||||
func (s *AdminServer) ListS3TablesTags(c *gin.Context) {
|
||||
resourceArn := c.Query("arn")
|
||||
if resourceArn == "" {
|
||||
c.JSON(400, gin.H{"error": "arn query parameter is required"})
|
||||
return
|
||||
}
|
||||
listReq := &s3tables.ListTagsForResourceRequest{ResourceARN: resourceArn}
|
||||
var resp s3tables.ListTagsForResourceResponse
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "ListTagsForResource", listReq, &resp); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, resp)
|
||||
}
|
||||
|
||||
func (s *AdminServer) UntagS3TablesResource(c *gin.Context) {
|
||||
var req struct {
|
||||
ResourceARN string `json:"resource_arn"`
|
||||
TagKeys []string `json:"tag_keys"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
if req.ResourceARN == "" || len(req.TagKeys) == 0 {
|
||||
c.JSON(400, gin.H{"error": "resource_arn and tag_keys are required"})
|
||||
return
|
||||
}
|
||||
untagReq := &s3tables.UntagResourceRequest{ResourceARN: req.ResourceARN, TagKeys: req.TagKeys}
|
||||
if err := s.executeS3TablesOperation(c.Request.Context(), "UntagResource", untagReq, nil); err != nil {
|
||||
writeS3TablesError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"message": "Tags removed"})
|
||||
}
|
||||
|
||||
func parseS3TablesErrorMessage(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
var s3Err *s3tables.S3TablesError
|
||||
if errors.As(err, &s3Err) {
|
||||
if s3Err.Message != "" {
|
||||
return fmt.Sprintf("%s: %s", s3Err.Type, s3Err.Message)
|
||||
}
|
||||
return s3Err.Type
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
func writeS3TablesError(c *gin.Context, err error) {
|
||||
c.JSON(s3TablesErrorStatus(err), gin.H{"error": parseS3TablesErrorMessage(err)})
|
||||
}
|
||||
|
||||
func s3TablesErrorStatus(err error) int {
|
||||
var s3Err *s3tables.S3TablesError
|
||||
if errors.As(err, &s3Err) {
|
||||
switch s3Err.Type {
|
||||
case s3tables.ErrCodeInvalidRequest:
|
||||
return http.StatusBadRequest
|
||||
case s3tables.ErrCodeNoSuchBucket, s3tables.ErrCodeNoSuchNamespace, s3tables.ErrCodeNoSuchTable, s3tables.ErrCodeNoSuchPolicy:
|
||||
return http.StatusNotFound
|
||||
case s3tables.ErrCodeAccessDenied:
|
||||
return http.StatusForbidden
|
||||
case s3tables.ErrCodeBucketAlreadyExists, s3tables.ErrCodeNamespaceAlreadyExists, s3tables.ErrCodeTableAlreadyExists, s3tables.ErrCodeConflict:
|
||||
return http.StatusConflict
|
||||
}
|
||||
}
|
||||
return http.StatusInternalServerError
|
||||
}
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
)
|
||||
|
||||
@@ -86,6 +88,9 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
|
||||
protected.GET("/object-store/users", h.userHandlers.ShowObjectStoreUsers)
|
||||
protected.GET("/object-store/policies", h.policyHandlers.ShowPolicies)
|
||||
protected.GET("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts)
|
||||
protected.GET("/object-store/s3tables/buckets", h.ShowS3TablesBuckets)
|
||||
protected.GET("/object-store/s3tables/buckets/:bucket/namespaces", h.ShowS3TablesNamespaces)
|
||||
protected.GET("/object-store/s3tables/buckets/:bucket/namespaces/:namespace/tables", h.ShowS3TablesTables)
|
||||
|
||||
// File browser routes
|
||||
protected.GET("/files", h.fileBrowserHandlers.ShowFileBrowser)
|
||||
@@ -174,6 +179,29 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
|
||||
objectStorePoliciesApi.POST("/validate", h.policyHandlers.ValidatePolicy)
|
||||
}
|
||||
|
||||
// S3 Tables API routes
|
||||
s3TablesApi := api.Group("/s3tables")
|
||||
{
|
||||
s3TablesApi.GET("/buckets", h.adminServer.ListS3TablesBucketsAPI)
|
||||
s3TablesApi.POST("/buckets", dash.RequireWriteAccess(), h.adminServer.CreateS3TablesBucket)
|
||||
s3TablesApi.DELETE("/buckets", dash.RequireWriteAccess(), h.adminServer.DeleteS3TablesBucket)
|
||||
s3TablesApi.GET("/namespaces", h.adminServer.ListS3TablesNamespacesAPI)
|
||||
s3TablesApi.POST("/namespaces", dash.RequireWriteAccess(), h.adminServer.CreateS3TablesNamespace)
|
||||
s3TablesApi.DELETE("/namespaces", dash.RequireWriteAccess(), h.adminServer.DeleteS3TablesNamespace)
|
||||
s3TablesApi.GET("/tables", h.adminServer.ListS3TablesTablesAPI)
|
||||
s3TablesApi.POST("/tables", dash.RequireWriteAccess(), h.adminServer.CreateS3TablesTable)
|
||||
s3TablesApi.DELETE("/tables", dash.RequireWriteAccess(), h.adminServer.DeleteS3TablesTable)
|
||||
s3TablesApi.PUT("/bucket-policy", dash.RequireWriteAccess(), h.adminServer.PutS3TablesBucketPolicy)
|
||||
s3TablesApi.GET("/bucket-policy", h.adminServer.GetS3TablesBucketPolicy)
|
||||
s3TablesApi.DELETE("/bucket-policy", dash.RequireWriteAccess(), h.adminServer.DeleteS3TablesBucketPolicy)
|
||||
s3TablesApi.PUT("/table-policy", dash.RequireWriteAccess(), h.adminServer.PutS3TablesTablePolicy)
|
||||
s3TablesApi.GET("/table-policy", h.adminServer.GetS3TablesTablePolicy)
|
||||
s3TablesApi.DELETE("/table-policy", dash.RequireWriteAccess(), h.adminServer.DeleteS3TablesTablePolicy)
|
||||
s3TablesApi.PUT("/tags", dash.RequireWriteAccess(), h.adminServer.TagS3TablesResource)
|
||||
s3TablesApi.GET("/tags", h.adminServer.ListS3TablesTags)
|
||||
s3TablesApi.DELETE("/tags", dash.RequireWriteAccess(), h.adminServer.UntagS3TablesResource)
|
||||
}
|
||||
|
||||
// File management API routes
|
||||
filesApi := api.Group("/files")
|
||||
{
|
||||
@@ -228,6 +256,9 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
|
||||
r.GET("/object-store/users", h.userHandlers.ShowObjectStoreUsers)
|
||||
r.GET("/object-store/policies", h.policyHandlers.ShowPolicies)
|
||||
r.GET("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts)
|
||||
r.GET("/object-store/s3tables/buckets", h.ShowS3TablesBuckets)
|
||||
r.GET("/object-store/s3tables/buckets/:bucket/namespaces", h.ShowS3TablesNamespaces)
|
||||
r.GET("/object-store/s3tables/buckets/:bucket/namespaces/:namespace/tables", h.ShowS3TablesTables)
|
||||
|
||||
// File browser routes
|
||||
r.GET("/files", h.fileBrowserHandlers.ShowFileBrowser)
|
||||
@@ -315,6 +346,29 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
|
||||
objectStorePoliciesApi.POST("/validate", h.policyHandlers.ValidatePolicy)
|
||||
}
|
||||
|
||||
// S3 Tables API routes
|
||||
s3TablesApi := api.Group("/s3tables")
|
||||
{
|
||||
s3TablesApi.GET("/buckets", h.adminServer.ListS3TablesBucketsAPI)
|
||||
s3TablesApi.POST("/buckets", h.adminServer.CreateS3TablesBucket)
|
||||
s3TablesApi.DELETE("/buckets", h.adminServer.DeleteS3TablesBucket)
|
||||
s3TablesApi.GET("/namespaces", h.adminServer.ListS3TablesNamespacesAPI)
|
||||
s3TablesApi.POST("/namespaces", h.adminServer.CreateS3TablesNamespace)
|
||||
s3TablesApi.DELETE("/namespaces", h.adminServer.DeleteS3TablesNamespace)
|
||||
s3TablesApi.GET("/tables", h.adminServer.ListS3TablesTablesAPI)
|
||||
s3TablesApi.POST("/tables", h.adminServer.CreateS3TablesTable)
|
||||
s3TablesApi.DELETE("/tables", h.adminServer.DeleteS3TablesTable)
|
||||
s3TablesApi.PUT("/bucket-policy", h.adminServer.PutS3TablesBucketPolicy)
|
||||
s3TablesApi.GET("/bucket-policy", h.adminServer.GetS3TablesBucketPolicy)
|
||||
s3TablesApi.DELETE("/bucket-policy", h.adminServer.DeleteS3TablesBucketPolicy)
|
||||
s3TablesApi.PUT("/table-policy", h.adminServer.PutS3TablesTablePolicy)
|
||||
s3TablesApi.GET("/table-policy", h.adminServer.GetS3TablesTablePolicy)
|
||||
s3TablesApi.DELETE("/table-policy", h.adminServer.DeleteS3TablesTablePolicy)
|
||||
s3TablesApi.PUT("/tags", h.adminServer.TagS3TablesResource)
|
||||
s3TablesApi.GET("/tags", h.adminServer.ListS3TablesTags)
|
||||
s3TablesApi.DELETE("/tags", h.adminServer.UntagS3TablesResource)
|
||||
}
|
||||
|
||||
// File management API routes
|
||||
filesApi := api.Group("/files")
|
||||
{
|
||||
@@ -398,6 +452,91 @@ func (h *AdminHandlers) ShowS3Buckets(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3TablesBuckets renders the S3 Tables buckets page
|
||||
func (h *AdminHandlers) ShowS3TablesBuckets(c *gin.Context) {
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
|
||||
data, err := h.adminServer.GetS3TablesBucketsData(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get S3 Tables buckets: " + err.Error()})
|
||||
return
|
||||
}
|
||||
data.Username = username
|
||||
|
||||
c.Header("Content-Type", "text/html")
|
||||
component := app.S3TablesBuckets(data)
|
||||
layoutComponent := layout.Layout(c, component)
|
||||
if err := layoutComponent.Render(c.Request.Context(), c.Writer); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3TablesNamespaces renders namespaces for a table bucket
|
||||
func (h *AdminHandlers) ShowS3TablesNamespaces(c *gin.Context) {
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
|
||||
bucketName := c.Param("bucket")
|
||||
arn, err := buildS3TablesBucketArn(bucketName)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
data, err := h.adminServer.GetS3TablesNamespacesData(c.Request.Context(), arn)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get S3 Tables namespaces: " + err.Error()})
|
||||
return
|
||||
}
|
||||
data.Username = username
|
||||
|
||||
c.Header("Content-Type", "text/html")
|
||||
component := app.S3TablesNamespaces(data)
|
||||
layoutComponent := layout.Layout(c, component)
|
||||
if err := layoutComponent.Render(c.Request.Context(), c.Writer); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
}
|
||||
}
|
||||
|
||||
// ShowS3TablesTables renders tables for a namespace
|
||||
func (h *AdminHandlers) ShowS3TablesTables(c *gin.Context) {
|
||||
username := c.GetString("username")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
|
||||
bucketName := c.Param("bucket")
|
||||
namespace := c.Param("namespace")
|
||||
arn, err := buildS3TablesBucketArn(bucketName)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
data, err := h.adminServer.GetS3TablesTablesData(c.Request.Context(), arn, namespace)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get S3 Tables tables: " + err.Error()})
|
||||
return
|
||||
}
|
||||
data.Username = username
|
||||
|
||||
c.Header("Content-Type", "text/html")
|
||||
component := app.S3TablesTables(data)
|
||||
layoutComponent := layout.Layout(c, component)
|
||||
if err := layoutComponent.Render(c.Request.Context(), c.Writer); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to render template: " + err.Error()})
|
||||
}
|
||||
}
|
||||
|
||||
func buildS3TablesBucketArn(bucketName string) (string, error) {
|
||||
return s3tables.BuildBucketARN(s3tables.DefaultRegion, s3_constants.AccountAdminId, bucketName)
|
||||
}
|
||||
|
||||
// ShowBucketDetails returns detailed information about a specific bucket
|
||||
func (h *AdminHandlers) ShowBucketDetails(c *gin.Context) {
|
||||
bucketName := c.Param("bucket")
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/http/client"
|
||||
)
|
||||
@@ -121,7 +122,6 @@ func (h *FileBrowserHandlers) DeleteFile(c *gin.Context) {
|
||||
})
|
||||
return err
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete file: " + err.Error()})
|
||||
return
|
||||
@@ -228,7 +228,7 @@ func (h *FileBrowserHandlers) CreateFolder(c *gin.Context) {
|
||||
Name: filepath.Base(fullPath),
|
||||
IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
FileMode: uint32(0755 | os.ModeDir), // Directory mode
|
||||
FileMode: uint32(0o755 | os.ModeDir), // Directory mode
|
||||
Uid: filer_pb.OS_UID,
|
||||
Gid: filer_pb.OS_GID,
|
||||
Crtime: time.Now().Unix(),
|
||||
@@ -239,7 +239,6 @@ func (h *FileBrowserHandlers) CreateFolder(c *gin.Context) {
|
||||
})
|
||||
return err
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create folder: " + err.Error()})
|
||||
return
|
||||
@@ -407,6 +406,9 @@ func (h *FileBrowserHandlers) uploadFileToFiler(filePath string, fileHeader *mul
|
||||
// Set content type with boundary
|
||||
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
|
||||
// Add JWT Token to Authorization Header
|
||||
h.setupFilerJwtAuth(req, "jwt.filer_signing.key", "jwt.filer_signing.expires_after_seconds", "filer upload")
|
||||
|
||||
// Send request using TLS-aware HTTP client with 60s timeout for large file uploads
|
||||
// lgtm[go/ssrf]
|
||||
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
|
||||
@@ -525,7 +527,12 @@ func (h *FileBrowserHandlers) fetchFileContent(filePath string, timeout time.Dur
|
||||
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
|
||||
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
|
||||
client := h.newClientWithTimeout(timeout)
|
||||
resp, err := client.Get(fileURL)
|
||||
req, err := http.NewRequest("GET", fileURL, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
h.addFilerJwtAuthHeader(req)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to fetch file from filer: %w", err)
|
||||
}
|
||||
@@ -595,6 +602,9 @@ func (h *FileBrowserHandlers) DownloadFile(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
client := h.newClientWithTimeout(5 * time.Minute) // Longer timeout for large file downloads
|
||||
|
||||
h.addFilerJwtAuthHeader(req)
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "Failed to fetch file from filer: " + err.Error()})
|
||||
@@ -687,7 +697,6 @@ func (h *FileBrowserHandlers) ViewFile(c *gin.Context) {
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get file metadata: " + err.Error()})
|
||||
return
|
||||
@@ -837,7 +846,6 @@ func (h *FileBrowserHandlers) GetFileProperties(c *gin.Context) {
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get file properties: " + err.Error()})
|
||||
return
|
||||
@@ -1032,7 +1040,13 @@ func (h *FileBrowserHandlers) isLikelyTextFile(filePath string, maxCheckSize int
|
||||
// Safe: filerAddress validated by validateFilerAddress() to match configured filer
|
||||
// Safe: cleanFilePath validated and cleaned by validateAndCleanFilePath() to prevent path traversal
|
||||
client := h.newClientWithTimeout(10 * time.Second)
|
||||
resp, err := client.Get(fileURL)
|
||||
req, err := http.NewRequest("GET", fileURL, nil)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to create request: %v", err)
|
||||
return false
|
||||
}
|
||||
h.addFilerJwtAuthHeader(req)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
@@ -1086,3 +1100,36 @@ func min(a, b int64) int64 {
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// setupFilerJwtAuth generates a JWT token and adds it to the request Authorization header if configured.
|
||||
func (h *FileBrowserHandlers) setupFilerJwtAuth(req *http.Request, keyPath, expiresPath, operation string) {
|
||||
// Load security configuration
|
||||
v := util.GetViper()
|
||||
|
||||
// Read Filer JWT token from security.toml
|
||||
signingKey := security.SigningKey(v.GetString(keyPath))
|
||||
expiresAfterSec := v.GetInt(expiresPath)
|
||||
|
||||
// Generate JWT token to authenticate with Filer
|
||||
var jwtToken security.EncodedJwt
|
||||
if len(signingKey) > 0 {
|
||||
jwtToken = security.GenJwtForFilerServer(signingKey, expiresAfterSec)
|
||||
glog.V(4).Infof("Generated JWT token for %s (expires in %d sec)", operation, expiresAfterSec)
|
||||
} else {
|
||||
if v.GetString("jwt.signing.key") != "" {
|
||||
glog.Warningf("JWT %s key not configured, but general JWT security is enabled. %s without authentication.", keyPath, operation)
|
||||
} else {
|
||||
glog.V(1).Infof("No JWT signing key configured, %s without authentication", operation)
|
||||
}
|
||||
}
|
||||
|
||||
// Add JWT Token to Authorization Header
|
||||
if jwtToken != "" {
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", string(jwtToken)))
|
||||
glog.V(4).Infof("Added JWT authorization header for %s", operation)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *FileBrowserHandlers) addFilerJwtAuthHeader(req *http.Request) {
|
||||
h.setupFilerJwtAuth(req, "jwt.filer_signing.read.key", "jwt.filer_signing.read.expires_after_seconds", "filer request")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,479 @@
|
||||
/**
|
||||
* Shared S3 Tables functionality for the SeaweedFS Admin Dashboard.
|
||||
*/
|
||||
|
||||
// Shared Modals
|
||||
let s3tablesBucketDeleteModal = null;
|
||||
let s3tablesBucketPolicyModal = null;
|
||||
let s3tablesNamespaceDeleteModal = null;
|
||||
let s3tablesTableDeleteModal = null;
|
||||
let s3tablesTablePolicyModal = null;
|
||||
let s3tablesTagsModal = null;
|
||||
|
||||
/**
|
||||
* Initialize S3 Tables Buckets Page
|
||||
*/
|
||||
function initS3TablesBuckets() {
|
||||
s3tablesBucketDeleteModal = new bootstrap.Modal(document.getElementById('deleteS3TablesBucketModal'));
|
||||
s3tablesBucketPolicyModal = new bootstrap.Modal(document.getElementById('s3tablesBucketPolicyModal'));
|
||||
s3tablesTagsModal = new bootstrap.Modal(document.getElementById('s3tablesTagsModal'));
|
||||
|
||||
const ownerSelect = document.getElementById('s3tablesBucketOwner');
|
||||
if (ownerSelect) {
|
||||
document.getElementById('createS3TablesBucketModal').addEventListener('show.bs.modal', async function () {
|
||||
if (ownerSelect.options.length <= 1) {
|
||||
try {
|
||||
const response = await fetch('/api/users');
|
||||
const data = await response.json();
|
||||
const users = data.users || [];
|
||||
users.forEach(user => {
|
||||
const option = document.createElement('option');
|
||||
option.value = user.username;
|
||||
option.textContent = user.username;
|
||||
ownerSelect.appendChild(option);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching users for owner dropdown:', error);
|
||||
ownerSelect.innerHTML = '<option value="">No owner (admin-only access)</option>';
|
||||
ownerSelect.selectedIndex = 0;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
document.querySelectorAll('.s3tables-delete-bucket-btn').forEach(button => {
|
||||
button.addEventListener('click', function () {
|
||||
document.getElementById('deleteS3TablesBucketName').textContent = this.dataset.bucketName || '';
|
||||
document.getElementById('deleteS3TablesBucketModal').dataset.bucketArn = this.dataset.bucketArn || '';
|
||||
s3tablesBucketDeleteModal.show();
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('.s3tables-bucket-policy-btn').forEach(button => {
|
||||
button.addEventListener('click', function () {
|
||||
const bucketArn = this.dataset.bucketArn || '';
|
||||
document.getElementById('s3tablesBucketPolicyArn').value = bucketArn;
|
||||
loadS3TablesBucketPolicy(bucketArn);
|
||||
s3tablesBucketPolicyModal.show();
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('.s3tables-tags-btn').forEach(button => {
|
||||
button.addEventListener('click', function () {
|
||||
const resourceArn = this.dataset.resourceArn || '';
|
||||
openS3TablesTags(resourceArn);
|
||||
});
|
||||
});
|
||||
|
||||
const createForm = document.getElementById('createS3TablesBucketForm');
|
||||
if (createForm) {
|
||||
createForm.addEventListener('submit', async function (e) {
|
||||
e.preventDefault();
|
||||
const name = document.getElementById('s3tablesBucketName').value.trim();
|
||||
const owner = ownerSelect.value;
|
||||
const tagsInput = document.getElementById('s3tablesBucketTags').value.trim();
|
||||
const tags = parseTagsInput(tagsInput);
|
||||
if (tags === null) return;
|
||||
const payload = { name: name, tags: tags, owner: owner };
|
||||
|
||||
try {
|
||||
const response = await fetch('/api/s3tables/buckets', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to create bucket');
|
||||
return;
|
||||
}
|
||||
alert('Bucket created successfully');
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
alert('Failed to create bucket: ' + error.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const policyForm = document.getElementById('s3tablesBucketPolicyForm');
|
||||
if (policyForm) {
|
||||
policyForm.addEventListener('submit', async function (e) {
|
||||
e.preventDefault();
|
||||
const bucketArn = document.getElementById('s3tablesBucketPolicyArn').value;
|
||||
const policy = document.getElementById('s3tablesBucketPolicyText').value.trim();
|
||||
if (!policy) {
|
||||
alert('Policy JSON is required');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const response = await fetch('/api/s3tables/bucket-policy', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ bucket_arn: bucketArn, policy: policy })
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to update policy');
|
||||
return;
|
||||
}
|
||||
alert('Policy updated');
|
||||
s3tablesBucketPolicyModal.hide();
|
||||
} catch (error) {
|
||||
alert('Failed to update policy: ' + error.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const tagsForm = document.getElementById('s3tablesTagsForm');
|
||||
if (tagsForm) {
|
||||
tagsForm.addEventListener('submit', async function (e) {
|
||||
e.preventDefault();
|
||||
const resourceArn = document.getElementById('s3tablesTagsResourceArn').value;
|
||||
const tags = parseTagsInput(document.getElementById('s3tablesTagsInput').value.trim());
|
||||
if (tags === null || Object.keys(tags).length === 0) {
|
||||
alert('Please provide tags to update');
|
||||
return;
|
||||
}
|
||||
await updateS3TablesTags(resourceArn, tags);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize S3 Tables Tables Page
|
||||
*/
|
||||
function initS3TablesTables() {
|
||||
s3tablesTableDeleteModal = new bootstrap.Modal(document.getElementById('deleteS3TablesTableModal'));
|
||||
s3tablesTablePolicyModal = new bootstrap.Modal(document.getElementById('s3tablesTablePolicyModal'));
|
||||
s3tablesTagsModal = new bootstrap.Modal(document.getElementById('s3tablesTagsModal'));
|
||||
|
||||
const dataContainer = document.getElementById('s3tables-tables-content');
|
||||
const dataBucketArn = dataContainer.dataset.bucketArn || '';
|
||||
const dataNamespace = dataContainer.dataset.namespace || '';
|
||||
|
||||
document.querySelectorAll('.s3tables-delete-table-btn').forEach(button => {
|
||||
button.addEventListener('click', function () {
|
||||
document.getElementById('deleteS3TablesTableName').textContent = this.dataset.tableName || '';
|
||||
document.getElementById('deleteS3TablesTableModal').dataset.tableName = this.dataset.tableName || '';
|
||||
s3tablesTableDeleteModal.show();
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('.s3tables-table-policy-btn').forEach(button => {
|
||||
button.addEventListener('click', function () {
|
||||
document.getElementById('s3tablesTablePolicyBucketArn').value = dataBucketArn;
|
||||
document.getElementById('s3tablesTablePolicyNamespace').value = dataNamespace;
|
||||
document.getElementById('s3tablesTablePolicyName').value = this.dataset.tableName || '';
|
||||
loadS3TablesTablePolicy(dataBucketArn, dataNamespace, this.dataset.tableName || '');
|
||||
s3tablesTablePolicyModal.show();
|
||||
});
|
||||
});
|
||||
|
||||
document.querySelectorAll('.s3tables-tags-btn').forEach(button => {
|
||||
button.addEventListener('click', function () {
|
||||
const resourceArn = this.dataset.resourceArn || '';
|
||||
openS3TablesTags(resourceArn);
|
||||
});
|
||||
});
|
||||
|
||||
const createForm = document.getElementById('createS3TablesTableForm');
|
||||
if (createForm) {
|
||||
createForm.addEventListener('submit', async function (e) {
|
||||
e.preventDefault();
|
||||
const name = document.getElementById('s3tablesTableName').value.trim();
|
||||
const format = document.getElementById('s3tablesTableFormat').value;
|
||||
const metadataText = document.getElementById('s3tablesTableMetadata').value.trim();
|
||||
const tagsInput = document.getElementById('s3tablesTableTags').value.trim();
|
||||
const tags = parseTagsInput(tagsInput);
|
||||
if (tags === null) return;
|
||||
let metadata = null;
|
||||
if (metadataText) {
|
||||
try {
|
||||
metadata = JSON.parse(metadataText);
|
||||
} catch (error) {
|
||||
alert('Invalid metadata JSON');
|
||||
return;
|
||||
}
|
||||
}
|
||||
const payload = { bucket_arn: dataBucketArn, namespace: dataNamespace, name: name, format: format, tags: tags };
|
||||
if (metadata) {
|
||||
payload.metadata = metadata;
|
||||
}
|
||||
try {
|
||||
const response = await fetch('/api/s3tables/tables', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(payload)
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to create table');
|
||||
return;
|
||||
}
|
||||
alert('Table created');
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
alert('Failed to create table: ' + error.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const policyForm = document.getElementById('s3tablesTablePolicyForm');
|
||||
if (policyForm) {
|
||||
policyForm.addEventListener('submit', async function (e) {
|
||||
e.preventDefault();
|
||||
const policy = document.getElementById('s3tablesTablePolicyText').value.trim();
|
||||
if (!policy) {
|
||||
alert('Policy JSON is required');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const response = await fetch('/api/s3tables/table-policy', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ bucket_arn: dataBucketArn, namespace: dataNamespace, name: document.getElementById('s3tablesTablePolicyName').value, policy: policy })
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to update policy');
|
||||
return;
|
||||
}
|
||||
alert('Policy updated');
|
||||
s3tablesTablePolicyModal.hide();
|
||||
} catch (error) {
|
||||
alert('Failed to update policy: ' + error.message);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const tagsForm = document.getElementById('s3tablesTagsForm');
|
||||
if (tagsForm) {
|
||||
tagsForm.addEventListener('submit', async function (e) {
|
||||
e.preventDefault();
|
||||
const resourceArn = document.getElementById('s3tablesTagsResourceArn').value;
|
||||
const tags = parseTagsInput(document.getElementById('s3tablesTagsInput').value.trim());
|
||||
if (tags === null || Object.keys(tags).length === 0) {
|
||||
alert('Please provide tags to update');
|
||||
return;
|
||||
}
|
||||
await updateS3TablesTags(resourceArn, tags);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Global scope functions used by onclick handlers
|
||||
|
||||
async function deleteS3TablesBucket() {
|
||||
const bucketArn = document.getElementById('deleteS3TablesBucketModal').dataset.bucketArn;
|
||||
if (!bucketArn) return;
|
||||
try {
|
||||
const response = await fetch(`/api/s3tables/buckets?bucket=${encodeURIComponent(bucketArn)}`, { method: 'DELETE' });
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to delete bucket');
|
||||
return;
|
||||
}
|
||||
alert('Bucket deleted');
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
alert('Failed to delete bucket: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadS3TablesBucketPolicy(bucketArn) {
|
||||
document.getElementById('s3tablesBucketPolicyText').value = '';
|
||||
if (!bucketArn) return;
|
||||
try {
|
||||
const response = await fetch(`/api/s3tables/bucket-policy?bucket=${encodeURIComponent(bucketArn)}`);
|
||||
const data = await response.json();
|
||||
if (response.ok && data.policy) {
|
||||
document.getElementById('s3tablesBucketPolicyText').value = data.policy;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load bucket policy', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteS3TablesBucketPolicy() {
|
||||
const bucketArn = document.getElementById('s3tablesBucketPolicyArn').value;
|
||||
if (!bucketArn) return;
|
||||
try {
|
||||
const response = await fetch(`/api/s3tables/bucket-policy?bucket=${encodeURIComponent(bucketArn)}`, { method: 'DELETE' });
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to delete policy');
|
||||
return;
|
||||
}
|
||||
alert('Policy deleted');
|
||||
document.getElementById('s3tablesBucketPolicyText').value = '';
|
||||
} catch (error) {
|
||||
alert('Failed to delete policy: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteS3TablesTable() {
|
||||
const dataContainer = document.getElementById('s3tables-tables-content');
|
||||
const dataBucketArn = dataContainer.dataset.bucketArn || '';
|
||||
const dataNamespace = dataContainer.dataset.namespace || '';
|
||||
const tableName = document.getElementById('deleteS3TablesTableModal').dataset.tableName;
|
||||
const versionToken = document.getElementById('deleteS3TablesTableVersion').value.trim();
|
||||
if (!tableName) return;
|
||||
const query = new URLSearchParams({
|
||||
bucket: dataBucketArn,
|
||||
namespace: dataNamespace,
|
||||
name: tableName
|
||||
});
|
||||
if (versionToken) {
|
||||
query.set('version', versionToken);
|
||||
}
|
||||
try {
|
||||
const response = await fetch(`/api/s3tables/tables?${query.toString()}`, { method: 'DELETE' });
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to delete table');
|
||||
return;
|
||||
}
|
||||
alert('Table deleted');
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
alert('Failed to delete table: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadS3TablesTablePolicy(bucketArn, namespace, name) {
|
||||
document.getElementById('s3tablesTablePolicyText').value = '';
|
||||
if (!bucketArn || !namespace || !name) return;
|
||||
const query = new URLSearchParams({ bucket: bucketArn, namespace: namespace, name: name });
|
||||
try {
|
||||
const response = await fetch(`/api/s3tables/table-policy?${query.toString()}`);
|
||||
const data = await response.json();
|
||||
if (response.ok && data.policy) {
|
||||
document.getElementById('s3tablesTablePolicyText').value = data.policy;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load table policy', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteS3TablesTablePolicy() {
|
||||
const dataContainer = document.getElementById('s3tables-tables-content');
|
||||
const dataBucketArn = dataContainer.dataset.bucketArn || '';
|
||||
const dataNamespace = dataContainer.dataset.namespace || '';
|
||||
const query = new URLSearchParams({ bucket: dataBucketArn, namespace: dataNamespace, name: document.getElementById('s3tablesTablePolicyName').value });
|
||||
try {
|
||||
const response = await fetch(`/api/s3tables/table-policy?${query.toString()}`, { method: 'DELETE' });
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to delete policy');
|
||||
return;
|
||||
}
|
||||
alert('Policy deleted');
|
||||
document.getElementById('s3tablesTablePolicyText').value = '';
|
||||
} catch (error) {
|
||||
alert('Failed to delete policy: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
function parseTagsInput(input) {
|
||||
if (!input) return {};
|
||||
const tags = {};
|
||||
const maxTags = 10;
|
||||
const maxKeyLength = 128;
|
||||
const maxValueLength = 256;
|
||||
const parts = input.split(',');
|
||||
for (const part of parts) {
|
||||
const trimmedPart = part.trim();
|
||||
if (!trimmedPart) continue;
|
||||
const idx = trimmedPart.indexOf('=');
|
||||
if (idx <= 0) {
|
||||
alert('Invalid tag format. Use key=value, and key cannot be empty.');
|
||||
return null;
|
||||
}
|
||||
const key = trimmedPart.slice(0, idx).trim();
|
||||
const value = trimmedPart.slice(idx + 1).trim();
|
||||
if (!key) {
|
||||
alert('Invalid tag format. Use key=value, and key cannot be empty.');
|
||||
return null;
|
||||
}
|
||||
if (key.length > maxKeyLength) {
|
||||
alert(`Tag key length must be <= ${maxKeyLength}`);
|
||||
return null;
|
||||
}
|
||||
if (value.length > maxValueLength) {
|
||||
alert(`Tag value length must be <= ${maxValueLength}`);
|
||||
return null;
|
||||
}
|
||||
tags[key] = value;
|
||||
if (Object.keys(tags).length > maxTags) {
|
||||
alert(`Too many tags. Max ${maxTags} tags allowed.`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return tags;
|
||||
}
|
||||
|
||||
async function openS3TablesTags(resourceArn) {
|
||||
if (!resourceArn) return;
|
||||
document.getElementById('s3tablesTagsResourceArn').value = resourceArn;
|
||||
document.getElementById('s3tablesTagsInput').value = '';
|
||||
document.getElementById('s3tablesTagsDeleteInput').value = '';
|
||||
document.getElementById('s3tablesTagsList').textContent = 'Loading...';
|
||||
s3tablesTagsModal.show();
|
||||
try {
|
||||
const response = await fetch(`/api/s3tables/tags?arn=${encodeURIComponent(resourceArn)}`);
|
||||
const data = await response.json();
|
||||
if (response.ok) {
|
||||
document.getElementById('s3tablesTagsList').textContent = JSON.stringify(data.tags || {}, null, 2);
|
||||
} else {
|
||||
document.getElementById('s3tablesTagsList').textContent = data.error || 'Failed to load tags';
|
||||
}
|
||||
} catch (error) {
|
||||
document.getElementById('s3tablesTagsList').textContent = error.message;
|
||||
}
|
||||
}
|
||||
|
||||
async function updateS3TablesTags(resourceArn, tags) {
|
||||
try {
|
||||
const response = await fetch('/api/s3tables/tags', {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ resource_arn: resourceArn, tags: tags })
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to update tags');
|
||||
return;
|
||||
}
|
||||
alert('Tags updated');
|
||||
openS3TablesTags(resourceArn);
|
||||
} catch (error) {
|
||||
alert('Failed to update tags: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function deleteS3TablesTags() {
|
||||
const resourceArn = document.getElementById('s3tablesTagsResourceArn').value;
|
||||
const keysInput = document.getElementById('s3tablesTagsDeleteInput').value.trim();
|
||||
if (!resourceArn) return;
|
||||
const tagKeys = keysInput.split(',').map(k => k.trim()).filter(k => k);
|
||||
if (tagKeys.length === 0) {
|
||||
alert('Provide tag keys to remove');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const response = await fetch('/api/s3tables/tags', {
|
||||
method: 'DELETE',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ resource_arn: resourceArn, tag_keys: tagKeys })
|
||||
});
|
||||
const data = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(data.error || 'Failed to remove tags');
|
||||
return;
|
||||
}
|
||||
alert('Tags removed');
|
||||
openS3TablesTags(resourceArn);
|
||||
} catch (error) {
|
||||
alert('Failed to remove tags: ' + error.message);
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,8 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
<h1 class="h2">
|
||||
if data.IsBucketPath && data.BucketName != "" {
|
||||
<i class="fas fa-cube me-2"></i>S3 Bucket: {data.BucketName}
|
||||
} else if data.IsTableBucketPath && data.TableBucketName != "" {
|
||||
<i class="fas fa-table me-2"></i>Table Bucket: {data.TableBucketName}
|
||||
} else {
|
||||
<i class="fas fa-folder-open me-2"></i>File Browser
|
||||
}
|
||||
@@ -26,6 +28,10 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
<a href="/object-store/buckets" class="btn btn-sm btn-outline-secondary">
|
||||
<i class="fas fa-arrow-left me-1"></i>Back to Buckets
|
||||
</a>
|
||||
} else if data.IsTableBucketPath && data.TableBucketName != "" {
|
||||
<a href="/object-store/s3tables/buckets" class="btn btn-sm btn-outline-secondary">
|
||||
<i class="fas fa-arrow-left me-1"></i>Back to Table Buckets
|
||||
</a>
|
||||
}
|
||||
<button type="button" class="btn btn-sm btn-outline-primary" onclick="createFolder()">
|
||||
<i class="fas fa-folder-plus me-1"></i>New Folder
|
||||
@@ -72,13 +78,18 @@ templ FileBrowser(data dash.FileBrowserData) {
|
||||
<div class="card-header py-3 d-flex justify-content-between align-items-center flex-wrap">
|
||||
<h6 class="m-0 font-weight-bold text-primary">
|
||||
<i class="fas fa-folder-open me-2"></i>
|
||||
if data.CurrentPath == "/" {
|
||||
if data.CurrentPath == "/" {
|
||||
<a href="/files?path=/" class="text-decoration-none text-primary">Root Directory</a>
|
||||
} else if data.CurrentPath == "/buckets" {
|
||||
<a href="/files?path=/buckets" class="text-decoration-none text-primary">Object Store Buckets Directory</a>
|
||||
<a href="/object-store/buckets" class="btn btn-sm btn-outline-primary ms-2">
|
||||
<i class="fas fa-cube me-1"></i>Manage Buckets
|
||||
</a>
|
||||
} else if data.CurrentPath == "/table-buckets" {
|
||||
<a href="/files?path=/table-buckets" class="text-decoration-none text-primary">Table Buckets Directory</a>
|
||||
<a href="/object-store/s3tables/buckets" class="btn btn-sm btn-outline-primary ms-2">
|
||||
<i class="fas fa-table me-1"></i>Manage Table Buckets
|
||||
</a>
|
||||
} else {
|
||||
<a href={ templ.SafeURL(fmt.Sprintf("/files?path=%s", data.CurrentPath)) } class="text-decoration-none text-primary">{ filepath.Base(data.CurrentPath) }</a>
|
||||
}
|
||||
@@ -767,4 +778,4 @@ func getMimeDisplayName(mime string) string {
|
||||
}
|
||||
return "File"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -220,6 +220,30 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
<option value="GetBucketObjectLockConfiguration">Get Bucket Object Lock Configuration</option>
|
||||
<option value="PutBucketObjectLockConfiguration">Put Bucket Object Lock Configuration</option>
|
||||
</optgroup>
|
||||
<optgroup label="S3 Tables Permissions">
|
||||
<option value="S3TablesAdmin">S3 Tables Admin (Full Access)</option>
|
||||
<option value="CreateTableBucket">Create Table Bucket</option>
|
||||
<option value="GetTableBucket">Get Table Bucket</option>
|
||||
<option value="ListTableBuckets">List Table Buckets</option>
|
||||
<option value="DeleteTableBucket">Delete Table Bucket</option>
|
||||
<option value="PutTableBucketPolicy">Put Table Bucket Policy</option>
|
||||
<option value="GetTableBucketPolicy">Get Table Bucket Policy</option>
|
||||
<option value="DeleteTableBucketPolicy">Delete Table Bucket Policy</option>
|
||||
<option value="CreateNamespace">Create Namespace</option>
|
||||
<option value="GetNamespace">Get Namespace</option>
|
||||
<option value="ListNamespaces">List Namespaces</option>
|
||||
<option value="DeleteNamespace">Delete Namespace</option>
|
||||
<option value="CreateTable">Create Table</option>
|
||||
<option value="GetTable">Get Table</option>
|
||||
<option value="ListTables">List Tables</option>
|
||||
<option value="DeleteTable">Delete Table</option>
|
||||
<option value="PutTablePolicy">Put Table Policy</option>
|
||||
<option value="GetTablePolicy">Get Table Policy</option>
|
||||
<option value="DeleteTablePolicy">Delete Table Policy</option>
|
||||
<option value="TagResource">Tag Resource</option>
|
||||
<option value="ListTagsForResource">List Tags</option>
|
||||
<option value="UntagResource">Untag Resource</option>
|
||||
</optgroup>
|
||||
</select>
|
||||
<small class="form-text text-muted">Hold Ctrl/Cmd to select multiple permissions</small>
|
||||
</div>
|
||||
@@ -304,6 +328,30 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
<option value="GetBucketObjectLockConfiguration">Get Bucket Object Lock Configuration</option>
|
||||
<option value="PutBucketObjectLockConfiguration">Put Bucket Object Lock Configuration</option>
|
||||
</optgroup>
|
||||
<optgroup label="S3 Tables Permissions">
|
||||
<option value="S3TablesAdmin">S3 Tables Admin (Full Access)</option>
|
||||
<option value="CreateTableBucket">Create Table Bucket</option>
|
||||
<option value="GetTableBucket">Get Table Bucket</option>
|
||||
<option value="ListTableBuckets">List Table Buckets</option>
|
||||
<option value="DeleteTableBucket">Delete Table Bucket</option>
|
||||
<option value="PutTableBucketPolicy">Put Table Bucket Policy</option>
|
||||
<option value="GetTableBucketPolicy">Get Table Bucket Policy</option>
|
||||
<option value="DeleteTableBucketPolicy">Delete Table Bucket Policy</option>
|
||||
<option value="CreateNamespace">Create Namespace</option>
|
||||
<option value="GetNamespace">Get Namespace</option>
|
||||
<option value="ListNamespaces">List Namespaces</option>
|
||||
<option value="DeleteNamespace">Delete Namespace</option>
|
||||
<option value="CreateTable">Create Table</option>
|
||||
<option value="GetTable">Get Table</option>
|
||||
<option value="ListTables">List Tables</option>
|
||||
<option value="DeleteTable">Delete Table</option>
|
||||
<option value="PutTablePolicy">Put Table Policy</option>
|
||||
<option value="GetTablePolicy">Get Table Policy</option>
|
||||
<option value="DeleteTablePolicy">Delete Table Policy</option>
|
||||
<option value="TagResource">Tag Resource</option>
|
||||
<option value="ListTagsForResource">List Tags</option>
|
||||
<option value="UntagResource">Untag Resource</option>
|
||||
</optgroup>
|
||||
</select>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
@@ -457,6 +505,32 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
// Global variable to store available buckets
|
||||
var availableBuckets = [];
|
||||
var bucketPermissionCounter = 0;
|
||||
const s3TablesPermissions = new Set([
|
||||
'CreateTableBucket',
|
||||
'GetTableBucket',
|
||||
'ListTableBuckets',
|
||||
'DeleteTableBucket',
|
||||
'PutTableBucketPolicy',
|
||||
'GetTableBucketPolicy',
|
||||
'DeleteTableBucketPolicy',
|
||||
'CreateNamespace',
|
||||
'GetNamespace',
|
||||
'ListNamespaces',
|
||||
'DeleteNamespace',
|
||||
'CreateTable',
|
||||
'GetTable',
|
||||
'ListTables',
|
||||
'DeleteTable',
|
||||
'PutTablePolicy',
|
||||
'GetTablePolicy',
|
||||
'DeleteTablePolicy',
|
||||
'TagResource',
|
||||
'ListTagsForResource',
|
||||
'UntagResource'
|
||||
]);
|
||||
function isS3TablesPermission(permission) {
|
||||
return permission === 'S3TablesAdmin' || s3TablesPermissions.has(permission);
|
||||
}
|
||||
|
||||
// Load buckets
|
||||
async function loadBuckets() {
|
||||
@@ -464,10 +538,8 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
const response = await fetch('/api/s3/buckets');
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
availableBuckets = data.buckets || [];
|
||||
availableBuckets = (data.buckets || []).map(bucket => ({ name: bucket.name, type: 's3' }));
|
||||
console.log('Loaded', availableBuckets.length, 'buckets');
|
||||
// Populate bucket selection dropdowns
|
||||
populateBucketSelections();
|
||||
} else {
|
||||
console.warn('Failed to load buckets');
|
||||
availableBuckets = [];
|
||||
@@ -476,6 +548,20 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
console.error('Error loading buckets:', error);
|
||||
availableBuckets = [];
|
||||
}
|
||||
try {
|
||||
const response = await fetch('/api/s3tables/buckets');
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
const tableBuckets = (data.buckets || data.tableBuckets || []).map(bucket => ({ name: bucket.name, type: 's3tables' }));
|
||||
availableBuckets = availableBuckets.concat(tableBuckets);
|
||||
} else {
|
||||
console.warn('Failed to load table buckets');
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Error loading table buckets:', error);
|
||||
}
|
||||
// Populate bucket selection dropdowns
|
||||
populateBucketSelections();
|
||||
}
|
||||
|
||||
// Load policies
|
||||
@@ -556,8 +642,8 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
select.innerHTML = '';
|
||||
availableBuckets.forEach(bucket => {
|
||||
const option = document.createElement('option');
|
||||
option.value = bucket.name;
|
||||
option.textContent = bucket.name;
|
||||
option.value = bucket.type + ':' + bucket.name;
|
||||
option.textContent = bucket.type === 's3tables' ? `Table: ${bucket.name}` : bucket.name;
|
||||
select.appendChild(option);
|
||||
});
|
||||
}
|
||||
@@ -584,11 +670,25 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
const globalBucketPerms = [];
|
||||
|
||||
actions.forEach(action => {
|
||||
if (action.includes(':')) {
|
||||
if (action.startsWith('s3tables:')) {
|
||||
const actionValue = action.slice('s3tables:'.length);
|
||||
if (actionValue === '*') {
|
||||
globalBucketPerms.push('S3TablesAdmin');
|
||||
return;
|
||||
}
|
||||
const parts = actionValue.split(':');
|
||||
const perm = parts[0];
|
||||
const bucket = parts.length > 1 ? parts.slice(1).join(':').replace(/\/\*$/, '') : '';
|
||||
if (bucket) {
|
||||
bucketActions.push({ permission: perm, bucketId: 's3tables:' + bucket });
|
||||
} else {
|
||||
globalBucketPerms.push(perm);
|
||||
}
|
||||
} else if (action.includes(':')) {
|
||||
const parts = action.split(':');
|
||||
const perm = parts[0];
|
||||
const bucket = parts.slice(1).join(':').replace(/\/\*$/, '');
|
||||
bucketActions.push({ permission: perm, bucket: bucket });
|
||||
bucketActions.push({ permission: perm, bucketId: 's3:' + bucket });
|
||||
} else {
|
||||
globalBucketPerms.push(action);
|
||||
}
|
||||
@@ -601,7 +701,7 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
} else if (bucketActions.length > 0) {
|
||||
// Get unique permissions and buckets
|
||||
const perms = [...new Set(bucketActions.map(ba => ba.permission))];
|
||||
const buckets = [...new Set(bucketActions.map(ba => ba.bucket))];
|
||||
const buckets = [...new Set(bucketActions.map(ba => ba.bucketId))];
|
||||
|
||||
result.permissions = perms;
|
||||
result.applyToAll = false;
|
||||
@@ -611,6 +711,16 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
return result;
|
||||
}
|
||||
|
||||
function parseBucketOptionValue(value) {
|
||||
if (value.startsWith('s3tables:')) {
|
||||
return { type: 's3tables', name: value.slice('s3tables:'.length) };
|
||||
}
|
||||
if (value.startsWith('s3:')) {
|
||||
return { type: 's3', name: value.slice('s3:'.length) };
|
||||
}
|
||||
return { type: 's3', name: value };
|
||||
}
|
||||
|
||||
// Build bucket permission action strings using original permissions dropdown
|
||||
/**
|
||||
* Builds bucket permission strings based on selected permissions and bucket scope.
|
||||
@@ -627,10 +737,8 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
// Get selected permissions from the original multi-select
|
||||
const selectedPerms = Array.from(permSelect.selectedOptions).map(opt => opt.value);
|
||||
|
||||
// If Admin is selected, return just Admin (it overrides everything)
|
||||
if (selectedPerms.includes('Admin')) {
|
||||
return ['Admin'];
|
||||
}
|
||||
const hasAdmin = selectedPerms.includes('Admin');
|
||||
const hasS3TablesAdmin = selectedPerms.includes('S3TablesAdmin');
|
||||
|
||||
if (selectedPerms.length === 0) {
|
||||
return [];
|
||||
@@ -663,13 +771,30 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
|
||||
if (applyToAll) {
|
||||
// Return global permissions (no bucket specification)
|
||||
return selectedPerms;
|
||||
const actions = [];
|
||||
if (hasAdmin) {
|
||||
actions.push('Admin');
|
||||
}
|
||||
if (hasS3TablesAdmin) {
|
||||
actions.push('s3tables:*');
|
||||
}
|
||||
selectedPerms.forEach(perm => {
|
||||
if (perm === 'Admin' || perm === 'S3TablesAdmin') {
|
||||
return;
|
||||
}
|
||||
if (isS3TablesPermission(perm)) {
|
||||
actions.push('s3tables:' + perm);
|
||||
} else {
|
||||
actions.push(perm);
|
||||
}
|
||||
});
|
||||
return actions;
|
||||
} else {
|
||||
// Get selected specific buckets
|
||||
const bucketSelect = document.getElementById(mode === 'edit' ? 'editSelectedBuckets' : 'selectedBuckets');
|
||||
if (!bucketSelect) return null;
|
||||
|
||||
const selectedBuckets = Array.from(bucketSelect.selectedOptions).map(opt => opt.value);
|
||||
const selectedBuckets = [...new Set(Array.from(bucketSelect.selectedOptions).map(opt => opt.value))];
|
||||
|
||||
// Return null to signal validation failure if no buckets selected
|
||||
if (selectedBuckets.length === 0) {
|
||||
@@ -678,13 +803,29 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
|
||||
// Build bucket-scoped permissions
|
||||
const actions = [];
|
||||
if (hasAdmin) {
|
||||
actions.push('Admin');
|
||||
}
|
||||
if (hasS3TablesAdmin) {
|
||||
actions.push('s3tables:*');
|
||||
}
|
||||
selectedPerms.forEach(perm => {
|
||||
if (perm === 'Admin' || perm === 'S3TablesAdmin') {
|
||||
return;
|
||||
}
|
||||
selectedBuckets.forEach(bucket => {
|
||||
actions.push(perm + ':' + bucket);
|
||||
const bucketInfo = parseBucketOptionValue(bucket);
|
||||
if (isS3TablesPermission(perm)) {
|
||||
if (bucketInfo.type === 's3tables') {
|
||||
actions.push('s3tables:' + perm + ':' + bucketInfo.name);
|
||||
}
|
||||
} else if (bucketInfo.type === 's3') {
|
||||
actions.push(perm + ':' + bucketInfo.name);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return actions;
|
||||
return [...new Set(actions)];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -834,6 +975,16 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
// Get permissions with bucket scope applied
|
||||
const allActions = buildBucketPermissions('create');
|
||||
|
||||
if (allActions === null) {
|
||||
showAlert('Please select at least one bucket when using specific bucket permissions', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
if (!allActions || allActions.length === 0) {
|
||||
showAlert('At least one permission must be selected', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
const userData = {
|
||||
username: formData.get('username'),
|
||||
email: formData.get('email'),
|
||||
@@ -887,15 +1038,15 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
// Get permissions with bucket scope applied
|
||||
const allActions = buildBucketPermissions('edit');
|
||||
|
||||
// Validate that permissions are not empty
|
||||
if (!allActions || allActions.length === 0) {
|
||||
showAlert('At least one permission must be selected', 'error');
|
||||
// Check for null (validation failure from buildBucketPermissions)
|
||||
if (allActions === null) {
|
||||
showAlert('Please select at least one bucket when using specific bucket permissions', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Check for null (validation failure from buildBucketPermissionsNew)
|
||||
if (allActions === null) {
|
||||
showAlert('Please select at least one bucket when using specific bucket permissions', 'error');
|
||||
// Validate that permissions are not empty
|
||||
if (!allActions || allActions.length === 0) {
|
||||
showAlert('At least one permission must be selected', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1154,4 +1305,4 @@ templ ObjectStoreUsers(data dash.ObjectStoreUsersData) {
|
||||
}
|
||||
|
||||
// Helper functions for template
|
||||
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,275 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
templ S3TablesBuckets(data dash.S3TablesBucketsData) {
|
||||
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
|
||||
<h1 class="h2">
|
||||
<i class="fas fa-table me-2"></i>S3 Tables Buckets
|
||||
</h1>
|
||||
<div class="btn-toolbar mb-2 mb-md-0">
|
||||
<div class="btn-group me-2">
|
||||
<button type="button" class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#createS3TablesBucketModal">
|
||||
<i class="fas fa-plus me-1"></i>Create Bucket
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="s3tables-buckets-content">
|
||||
<div class="row mb-4">
|
||||
<div class="col-xl-4 col-md-6 mb-4">
|
||||
<div class="card border-left-primary shadow h-100 py-2">
|
||||
<div class="card-body">
|
||||
<div class="row no-gutters align-items-center">
|
||||
<div class="col mr-2">
|
||||
<div class="text-xs font-weight-bold text-primary text-uppercase mb-1">
|
||||
Total Buckets
|
||||
</div>
|
||||
<div class="h5 mb-0 font-weight-bold text-gray-800">
|
||||
{ fmt.Sprintf("%d", data.TotalBuckets) }
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<i class="fas fa-table fa-2x text-gray-300"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-xl-4 col-md-6 mb-4">
|
||||
<div class="card border-left-info shadow h-100 py-2">
|
||||
<div class="card-body">
|
||||
<div class="row no-gutters align-items-center">
|
||||
<div class="col mr-2">
|
||||
<div class="text-xs font-weight-bold text-info text-uppercase mb-1">
|
||||
Last Updated
|
||||
</div>
|
||||
<div class="h6 mb-0 font-weight-bold text-gray-800">
|
||||
{ data.LastUpdated.Format("15:04") }
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<i class="fas fa-clock fa-2x text-gray-300"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-12">
|
||||
<div class="card shadow mb-4">
|
||||
<div class="card-header py-3 d-flex flex-row align-items-center justify-content-between">
|
||||
<h6 class="m-0 font-weight-bold text-primary">
|
||||
<i class="fas fa-table me-2"></i>Table Buckets
|
||||
</h6>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover" width="100%" cellspacing="0" id="s3tablesBucketsTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Owner</th>
|
||||
<th>ARN</th>
|
||||
<th>Created</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
for _, bucket := range data.Buckets {
|
||||
<tr>
|
||||
<td>{ bucket.Name }</td>
|
||||
<td>{ bucket.OwnerAccountID }</td>
|
||||
<td class="text-muted small">{ bucket.ARN }</td>
|
||||
<td>{ bucket.CreatedAt.Format("2006-01-02 15:04") }</td>
|
||||
<td>
|
||||
<div class="btn-group btn-group-sm" role="group">
|
||||
{{ bucketName, parseErr := s3tables.ParseBucketNameFromARN(bucket.ARN) }}
|
||||
if parseErr == nil {
|
||||
<a class="btn btn-outline-primary btn-sm" href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)) }>
|
||||
<i class="fas fa-folder-open"></i>
|
||||
</a>
|
||||
} else {
|
||||
<button type="button" class="btn btn-outline-primary btn-sm" disabled title="Invalid bucket ARN">
|
||||
<i class="fas fa-folder-open"></i>
|
||||
</button>
|
||||
}
|
||||
<button type="button" class="btn btn-outline-success btn-sm s3tables-tags-btn" data-resource-arn={ bucket.ARN } title="Tags">
|
||||
<i class="fas fa-tags"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-info btn-sm s3tables-bucket-policy-btn" data-bucket-arn={ bucket.ARN } title="Bucket Policy">
|
||||
<i class="fas fa-shield-alt"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-danger btn-sm s3tables-delete-bucket-btn" data-bucket-arn={ bucket.ARN } data-bucket-name={ bucket.Name } title="Delete">
|
||||
<i class="fas fa-trash"></i>
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
if len(data.Buckets) == 0 {
|
||||
<tr>
|
||||
<td colspan="5" class="text-center text-muted py-4">
|
||||
<i class="fas fa-table fa-3x mb-3 text-muted"></i>
|
||||
<div>
|
||||
<h5>No table buckets found</h5>
|
||||
<p>Create your first S3 Tables bucket to get started.</p>
|
||||
<button type="button" class="btn btn-primary" data-bs-toggle="modal" data-bs-target="#createS3TablesBucketModal">
|
||||
<i class="fas fa-plus me-1"></i>Create Bucket
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="createS3TablesBucketModal" tabindex="-1" aria-labelledby="createS3TablesBucketModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="createS3TablesBucketModalLabel">
|
||||
<i class="fas fa-plus me-2"></i>Create Table Bucket
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form id="createS3TablesBucketForm">
|
||||
<div class="modal-body">
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesBucketName" class="form-label">Bucket Name</label>
|
||||
<input type="text" class="form-control" id="s3tablesBucketName" name="name" placeholder="table-bucket-name" required/>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesBucketOwner" class="form-label">Owner (Optional)</label>
|
||||
<select class="form-select" id="s3tablesBucketOwner" name="owner">
|
||||
<option value="">No owner (admin-only access)</option>
|
||||
</select>
|
||||
<div class="form-text">
|
||||
The S3 identity that owns this table bucket. Non-admin users can only access table buckets they own.
|
||||
</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesBucketTags" class="form-label">Tags</label>
|
||||
<input type="text" class="form-control" id="s3tablesBucketTags" name="tags" placeholder="key1=value1,key2=value2"/>
|
||||
<div class="form-text">Optional tags in key=value format.</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<i class="fas fa-plus me-1"></i>Create
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="deleteS3TablesBucketModal" tabindex="-1" aria-labelledby="deleteS3TablesBucketModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="deleteS3TablesBucketModalLabel">
|
||||
<i class="fas fa-exclamation-triangle me-2 text-warning"></i>Delete Table Bucket
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Are you sure you want to delete the table bucket <strong id="deleteS3TablesBucketName"></strong>?</p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="button" class="btn btn-danger" onclick="deleteS3TablesBucket()">
|
||||
<i class="fas fa-trash me-1"></i>Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="s3tablesBucketPolicyModal" tabindex="-1" aria-labelledby="s3tablesBucketPolicyModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="s3tablesBucketPolicyModalLabel">
|
||||
<i class="fas fa-shield-alt me-2"></i>Table Bucket Policy
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form id="s3tablesBucketPolicyForm">
|
||||
<div class="modal-body">
|
||||
<input type="hidden" id="s3tablesBucketPolicyArn" name="bucket_arn"/>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesBucketPolicyText" class="form-label">Policy JSON</label>
|
||||
<textarea class="form-control" id="s3tablesBucketPolicyText" name="policy" rows="12" placeholder="{ }"></textarea>
|
||||
</div>
|
||||
<div class="form-text">
|
||||
Provide a policy JSON; use Delete Policy to remove the policy.
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
|
||||
<button type="button" class="btn btn-outline-danger" onclick="deleteS3TablesBucketPolicy()">
|
||||
<i class="fas fa-trash me-1"></i>Delete Policy
|
||||
</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<i class="fas fa-save me-1"></i>Save Policy
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="s3tablesTagsModal" tabindex="-1" aria-labelledby="s3tablesTagsModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="s3tablesTagsModalLabel">
|
||||
<i class="fas fa-tags me-2"></i>Resource Tags
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form id="s3tablesTagsForm">
|
||||
<div class="modal-body">
|
||||
<input type="hidden" id="s3tablesTagsResourceArn" name="resource_arn"/>
|
||||
<div class="mb-3">
|
||||
<label class="form-label">Existing Tags</label>
|
||||
<pre class="bg-light p-3 border rounded" id="s3tablesTagsList">Loading...</pre>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesTagsInput" class="form-label">Add or Update Tags</label>
|
||||
<input type="text" class="form-control" id="s3tablesTagsInput" placeholder="key1=value1,key2=value2"/>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesTagsDeleteInput" class="form-label">Remove Tag Keys</label>
|
||||
<input type="text" class="form-control" id="s3tablesTagsDeleteInput" placeholder="key1,key2"/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
|
||||
<button type="button" class="btn btn-outline-danger" onclick="deleteS3TablesTags()">
|
||||
<i class="fas fa-trash me-1"></i>Remove Tags
|
||||
</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<i class="fas fa-save me-1"></i>Update Tags
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
initS3TablesBuckets();
|
||||
});
|
||||
</script>
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,242 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
templ S3TablesNamespaces(data dash.S3TablesNamespacesData) {
|
||||
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
|
||||
<h1 class="h2">
|
||||
<i class="fas fa-layer-group me-2"></i>S3 Tables Namespaces
|
||||
</h1>
|
||||
<div class="btn-toolbar mb-2 mb-md-0">
|
||||
<div class="btn-group me-2">
|
||||
<button type="button" class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#createS3TablesNamespaceModal">
|
||||
<i class="fas fa-plus me-1"></i>Create Namespace
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<a href="/object-store/s3tables/buckets" class="btn btn-sm btn-outline-secondary">
|
||||
<i class="fas fa-arrow-left me-1"></i>Back to Buckets
|
||||
</a>
|
||||
<span class="text-muted ms-2">Bucket ARN: { data.BucketARN }</span>
|
||||
</div>
|
||||
<div id="s3tables-namespaces-content" data-bucket-arn={ data.BucketARN }>
|
||||
<div class="row mb-4">
|
||||
<div class="col-xl-4 col-md-6 mb-4">
|
||||
<div class="card border-left-primary shadow h-100 py-2">
|
||||
<div class="card-body">
|
||||
<div class="row no-gutters align-items-center">
|
||||
<div class="col mr-2">
|
||||
<div class="text-xs font-weight-bold text-primary text-uppercase mb-1">
|
||||
Total Namespaces
|
||||
</div>
|
||||
<div class="h5 mb-0 font-weight-bold text-gray-800">
|
||||
{ fmt.Sprintf("%d", data.TotalNamespaces) }
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<i class="fas fa-layer-group fa-2x text-gray-300"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-xl-4 col-md-6 mb-4">
|
||||
<div class="card border-left-info shadow h-100 py-2">
|
||||
<div class="card-body">
|
||||
<div class="row no-gutters align-items-center">
|
||||
<div class="col mr-2">
|
||||
<div class="text-xs font-weight-bold text-info text-uppercase mb-1">
|
||||
Last Updated
|
||||
</div>
|
||||
<div class="h6 mb-0 font-weight-bold text-gray-800">
|
||||
{ data.LastUpdated.Format("15:04") }
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<i class="fas fa-clock fa-2x text-gray-300"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-12">
|
||||
<div class="card shadow mb-4">
|
||||
<div class="card-header py-3 d-flex flex-row align-items-center justify-content-between">
|
||||
<h6 class="m-0 font-weight-bold text-primary">
|
||||
<i class="fas fa-layer-group me-2"></i>Namespaces
|
||||
</h6>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover" width="100%" cellspacing="0" id="s3tablesNamespacesTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Namespace</th>
|
||||
<th>Created</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
for _, namespace := range data.Namespaces {
|
||||
<tr>
|
||||
<td>{ strings.Join(namespace.Namespace, ".") }</td>
|
||||
<td>{ namespace.CreatedAt.Format("2006-01-02 15:04") }</td>
|
||||
<td>
|
||||
<div class="btn-group btn-group-sm" role="group">
|
||||
{{ bucketName, parseErr := s3tables.ParseBucketNameFromARN(data.BucketARN) }}
|
||||
{{ namespaceName := strings.Join(namespace.Namespace, ".") }}
|
||||
if parseErr == nil {
|
||||
<a class="btn btn-outline-primary btn-sm" href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables", bucketName, namespaceName)) }>
|
||||
<i class="fas fa-table"></i>
|
||||
</a>
|
||||
} else {
|
||||
<button type="button" class="btn btn-outline-primary btn-sm" disabled title="Invalid bucket ARN">
|
||||
<i class="fas fa-table"></i>
|
||||
</button>
|
||||
}
|
||||
<button type="button" class="btn btn-outline-danger btn-sm s3tables-delete-namespace-btn" data-namespace-name={ namespaceName } title="Delete">
|
||||
<i class="fas fa-trash"></i>
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
if len(data.Namespaces) == 0 {
|
||||
<tr>
|
||||
<td colspan="3" class="text-center text-muted py-4">
|
||||
<i class="fas fa-layer-group fa-3x mb-3 text-muted"></i>
|
||||
<div>
|
||||
<h5>No namespaces found</h5>
|
||||
<p>Create your first namespace to organize tables.</p>
|
||||
<button type="button" class="btn btn-primary" data-bs-toggle="modal" data-bs-target="#createS3TablesNamespaceModal">
|
||||
<i class="fas fa-plus me-1"></i>Create Namespace
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="createS3TablesNamespaceModal" tabindex="-1" aria-labelledby="createS3TablesNamespaceModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="createS3TablesNamespaceModalLabel">
|
||||
<i class="fas fa-plus me-2"></i>Create Namespace
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form id="createS3TablesNamespaceForm">
|
||||
<div class="modal-body">
|
||||
<input type="hidden" name="bucket_arn" value={ data.BucketARN }/>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesNamespaceName" class="form-label">Namespace</label>
|
||||
<input type="text" class="form-control" id="s3tablesNamespaceName" name="name" placeholder="analytics" required/>
|
||||
<div class="form-text">Namespaces use a single level (no dots).</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<i class="fas fa-plus me-1"></i>Create
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="deleteS3TablesNamespaceModal" tabindex="-1" aria-labelledby="deleteS3TablesNamespaceModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="deleteS3TablesNamespaceModalLabel">
|
||||
<i class="fas fa-exclamation-triangle me-2 text-warning"></i>Delete Namespace
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Are you sure you want to delete the namespace <strong id="deleteS3TablesNamespaceName"></strong>?</p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="button" class="btn btn-danger" onclick="deleteS3TablesNamespace()">
|
||||
<i class="fas fa-trash me-1"></i>Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
let s3tablesNamespaceDeleteModal = null;
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
s3tablesNamespaceDeleteModal = new bootstrap.Modal(document.getElementById('deleteS3TablesNamespaceModal'));
|
||||
|
||||
document.querySelectorAll('.s3tables-delete-namespace-btn').forEach(button => {
|
||||
button.addEventListener('click', function() {
|
||||
document.getElementById('deleteS3TablesNamespaceName').textContent = this.dataset.namespaceName || '';
|
||||
document.getElementById('deleteS3TablesNamespaceModal').dataset.namespaceName = this.dataset.namespaceName || '';
|
||||
s3tablesNamespaceDeleteModal.show();
|
||||
});
|
||||
});
|
||||
|
||||
document.getElementById('createS3TablesNamespaceForm').addEventListener('submit', async function(e) {
|
||||
e.preventDefault();
|
||||
const name = document.getElementById('s3tablesNamespaceName').value.trim();
|
||||
try {
|
||||
const response = await fetch('/api/s3tables/namespaces', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ bucket_arn: dataBucketArn, name: name })
|
||||
});
|
||||
const payload = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(payload.error || 'Failed to create namespace');
|
||||
return;
|
||||
}
|
||||
alert('Namespace created');
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
alert('Failed to create namespace: ' + error.message);
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
const dataBucketArn = document.getElementById('s3tables-namespaces-content').dataset.bucketArn || '';
|
||||
|
||||
async function deleteS3TablesNamespace() {
|
||||
const namespace = document.getElementById('deleteS3TablesNamespaceModal').dataset.namespaceName;
|
||||
if (!namespace) return;
|
||||
try {
|
||||
const response = await fetch(`/api/s3tables/namespaces?bucket=${encodeURIComponent(dataBucketArn)}&name=${encodeURIComponent(namespace)}`, { method: 'DELETE' });
|
||||
const payload = await response.json();
|
||||
if (!response.ok) {
|
||||
alert(payload.error || 'Failed to delete namespace');
|
||||
return;
|
||||
}
|
||||
alert('Namespace deleted');
|
||||
location.reload();
|
||||
} catch (error) {
|
||||
alert('Failed to delete namespace: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
// Code generated by templ - DO NOT EDIT.
|
||||
|
||||
// templ: version: v0.3.960
|
||||
package app
|
||||
|
||||
//lint:file-ignore SA4006 This context is only used if a nested component is present.
|
||||
|
||||
import "github.com/a-h/templ"
|
||||
import templruntime "github.com/a-h/templ/runtime"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
func S3TablesNamespaces(data dash.S3TablesNamespacesData) templ.Component {
|
||||
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
|
||||
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
|
||||
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
|
||||
return templ_7745c5c3_CtxErr
|
||||
}
|
||||
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
|
||||
if !templ_7745c5c3_IsBuffer {
|
||||
defer func() {
|
||||
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
|
||||
if templ_7745c5c3_Err == nil {
|
||||
templ_7745c5c3_Err = templ_7745c5c3_BufErr
|
||||
}
|
||||
}()
|
||||
}
|
||||
ctx = templ.InitializeContext(ctx)
|
||||
templ_7745c5c3_Var1 := templ.GetChildren(ctx)
|
||||
if templ_7745c5c3_Var1 == nil {
|
||||
templ_7745c5c3_Var1 = templ.NopComponent
|
||||
}
|
||||
ctx = templ.ClearChildren(ctx)
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><h1 class=\"h2\"><i class=\"fas fa-layer-group me-2\"></i>S3 Tables Namespaces</h1><div class=\"btn-toolbar mb-2 mb-md-0\"><div class=\"btn-group me-2\"><button type=\"button\" class=\"btn btn-sm btn-primary\" data-bs-toggle=\"modal\" data-bs-target=\"#createS3TablesNamespaceModal\"><i class=\"fas fa-plus me-1\"></i>Create Namespace</button></div></div></div><div class=\"mb-3\"><a href=\"/object-store/s3tables/buckets\" class=\"btn btn-sm btn-outline-secondary\"><i class=\"fas fa-arrow-left me-1\"></i>Back to Buckets</a> <span class=\"text-muted ms-2\">Bucket ARN: ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var2 string
|
||||
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(data.BucketARN)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_namespaces.templ`, Line: 28, Col: 60}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "</span></div><div id=\"s3tables-namespaces-content\" data-bucket-arn=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var3 string
|
||||
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(data.BucketARN)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_namespaces.templ`, Line: 30, Col: 71}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "\"><div class=\"row mb-4\"><div class=\"col-xl-4 col-md-6 mb-4\"><div class=\"card border-left-primary shadow h-100 py-2\"><div class=\"card-body\"><div class=\"row no-gutters align-items-center\"><div class=\"col mr-2\"><div class=\"text-xs font-weight-bold text-primary text-uppercase mb-1\">Total Namespaces</div><div class=\"h5 mb-0 font-weight-bold text-gray-800\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var4 string
|
||||
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", data.TotalNamespaces))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_namespaces.templ`, Line: 41, Col: 50}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "</div></div><div class=\"col-auto\"><i class=\"fas fa-layer-group fa-2x text-gray-300\"></i></div></div></div></div></div><div class=\"col-xl-4 col-md-6 mb-4\"><div class=\"card border-left-info shadow h-100 py-2\"><div class=\"card-body\"><div class=\"row no-gutters align-items-center\"><div class=\"col mr-2\"><div class=\"text-xs font-weight-bold text-info text-uppercase mb-1\">Last Updated</div><div class=\"h6 mb-0 font-weight-bold text-gray-800\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var5 string
|
||||
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(data.LastUpdated.Format("15:04"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_namespaces.templ`, Line: 60, Col: 43}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</div></div><div class=\"col-auto\"><i class=\"fas fa-clock fa-2x text-gray-300\"></i></div></div></div></div></div></div><div class=\"row\"><div class=\"col-12\"><div class=\"card shadow mb-4\"><div class=\"card-header py-3 d-flex flex-row align-items-center justify-content-between\"><h6 class=\"m-0 font-weight-bold text-primary\"><i class=\"fas fa-layer-group me-2\"></i>Namespaces</h6></div><div class=\"card-body\"><div class=\"table-responsive\"><table class=\"table table-hover\" width=\"100%\" cellspacing=\"0\" id=\"s3tablesNamespacesTable\"><thead><tr><th>Namespace</th><th>Created</th><th>Actions</th></tr></thead> <tbody>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
for _, namespace := range data.Namespaces {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<tr><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var6 string
|
||||
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(strings.Join(namespace.Namespace, "."))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_namespaces.templ`, Line: 92, Col: 55}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</td><td>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var7 string
|
||||
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(namespace.CreatedAt.Format("2006-01-02 15:04"))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_namespaces.templ`, Line: 93, Col: 63}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</td><td><div class=\"btn-group btn-group-sm\" role=\"group\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
bucketName, parseErr := s3tables.ParseBucketNameFromARN(data.BucketARN)
|
||||
namespaceName := strings.Join(namespace.Namespace, ".")
|
||||
if parseErr == nil {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "<a class=\"btn btn-outline-primary btn-sm\" href=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var8 templ.SafeURL
|
||||
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces/%s/tables", bucketName, namespaceName)))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_namespaces.templ`, Line: 99, Col: 174}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "\"><i class=\"fas fa-table\"></i></a> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
} else {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "<button type=\"button\" class=\"btn btn-outline-primary btn-sm\" disabled title=\"Invalid bucket ARN\"><i class=\"fas fa-table\"></i></button> ")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "<button type=\"button\" class=\"btn btn-outline-danger btn-sm s3tables-delete-namespace-btn\" data-namespace-name=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var9 string
|
||||
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(namespaceName)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_namespaces.templ`, Line: 107, Col: 138}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "\" title=\"Delete\"><i class=\"fas fa-trash\"></i></button></div></td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
if len(data.Namespaces) == 0 {
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "<tr><td colspan=\"3\" class=\"text-center text-muted py-4\"><i class=\"fas fa-layer-group fa-3x mb-3 text-muted\"></i><div><h5>No namespaces found</h5><p>Create your first namespace to organize tables.</p><button type=\"button\" class=\"btn btn-primary\" data-bs-toggle=\"modal\" data-bs-target=\"#createS3TablesNamespaceModal\"><i class=\"fas fa-plus me-1\"></i>Create Namespace</button></div></td></tr>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "</tbody></table></div></div></div></div></div></div><div class=\"modal fade\" id=\"createS3TablesNamespaceModal\" tabindex=\"-1\" aria-labelledby=\"createS3TablesNamespaceModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"createS3TablesNamespaceModalLabel\"><i class=\"fas fa-plus me-2\"></i>Create Namespace</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><form id=\"createS3TablesNamespaceForm\"><div class=\"modal-body\"><input type=\"hidden\" name=\"bucket_arn\" value=\"")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
var templ_7745c5c3_Var10 string
|
||||
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(data.BucketARN)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/s3tables_namespaces.templ`, Line: 147, Col: 66}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "\"><div class=\"mb-3\"><label for=\"s3tablesNamespaceName\" class=\"form-label\">Namespace</label> <input type=\"text\" class=\"form-control\" id=\"s3tablesNamespaceName\" name=\"name\" placeholder=\"analytics\" required><div class=\"form-text\">Namespaces use a single level (no dots).</div></div></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"submit\" class=\"btn btn-primary\"><i class=\"fas fa-plus me-1\"></i>Create</button></div></form></div></div></div><div class=\"modal fade\" id=\"deleteS3TablesNamespaceModal\" tabindex=\"-1\" aria-labelledby=\"deleteS3TablesNamespaceModalLabel\" aria-hidden=\"true\"><div class=\"modal-dialog\"><div class=\"modal-content\"><div class=\"modal-header\"><h5 class=\"modal-title\" id=\"deleteS3TablesNamespaceModalLabel\"><i class=\"fas fa-exclamation-triangle me-2 text-warning\"></i>Delete Namespace</h5><button type=\"button\" class=\"btn-close\" data-bs-dismiss=\"modal\" aria-label=\"Close\"></button></div><div class=\"modal-body\"><p>Are you sure you want to delete the namespace <strong id=\"deleteS3TablesNamespaceName\"></strong>?</p></div><div class=\"modal-footer\"><button type=\"button\" class=\"btn btn-secondary\" data-bs-dismiss=\"modal\">Cancel</button> <button type=\"button\" class=\"btn btn-danger\" onclick=\"deleteS3TablesNamespace()\"><i class=\"fas fa-trash me-1\"></i>Delete</button></div></div></div></div><script>\n\t\tlet s3tablesNamespaceDeleteModal = null;\n\n\t\tdocument.addEventListener('DOMContentLoaded', function() {\n\t\t\ts3tablesNamespaceDeleteModal = new bootstrap.Modal(document.getElementById('deleteS3TablesNamespaceModal'));\n\n\t\t\tdocument.querySelectorAll('.s3tables-delete-namespace-btn').forEach(button => {\n\t\t\t\tbutton.addEventListener('click', function() {\n\t\t\t\t\tdocument.getElementById('deleteS3TablesNamespaceName').textContent = this.dataset.namespaceName || '';\n\t\t\t\t\tdocument.getElementById('deleteS3TablesNamespaceModal').dataset.namespaceName = this.dataset.namespaceName || '';\n\t\t\t\t\ts3tablesNamespaceDeleteModal.show();\n\t\t\t\t});\n\t\t\t});\n\n\t\t\tdocument.getElementById('createS3TablesNamespaceForm').addEventListener('submit', async function(e) {\n\t\t\t\te.preventDefault();\n\t\t\t\tconst name = document.getElementById('s3tablesNamespaceName').value.trim();\n\t\t\t\ttry {\n\t\t\t\t\t\tconst response = await fetch('/api/s3tables/namespaces', {\n\t\t\t\t\t\t\tmethod: 'POST',\n\t\t\t\t\t\t\theaders: { 'Content-Type': 'application/json' },\n\t\t\t\t\t\t\tbody: JSON.stringify({ bucket_arn: dataBucketArn, name: name })\n\t\t\t\t\t\t});\n\t\t\t\t\tconst payload = await response.json();\n\t\t\t\t\tif (!response.ok) {\n\t\t\t\t\t\talert(payload.error || 'Failed to create namespace');\n\t\t\t\t\t\treturn;\n\t\t\t\t\t}\n\t\t\t\t\talert('Namespace created');\n\t\t\t\t\tlocation.reload();\n\t\t\t\t} catch (error) {\n\t\t\t\t\talert('Failed to create namespace: ' + error.message);\n\t\t\t\t}\n\t\t\t});\n\n\t\t});\n\n\t\tconst dataBucketArn = document.getElementById('s3tables-namespaces-content').dataset.bucketArn || '';\n\n\t\tasync function deleteS3TablesNamespace() {\n\t\t\tconst namespace = document.getElementById('deleteS3TablesNamespaceModal').dataset.namespaceName;\n\t\t\tif (!namespace) return;\n\t\t\ttry {\n\t\t\t\tconst response = await fetch(`/api/s3tables/namespaces?bucket=${encodeURIComponent(dataBucketArn)}&name=${encodeURIComponent(namespace)}`, { method: 'DELETE' });\n\t\t\t\tconst payload = await response.json();\n\t\t\t\tif (!response.ok) {\n\t\t\t\t\talert(payload.error || 'Failed to delete namespace');\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t\talert('Namespace deleted');\n\t\t\t\tlocation.reload();\n\t\t\t} catch (error) {\n\t\t\t\talert('Failed to delete namespace: ' + error.message);\n\t\t\t}\n\t\t}\n\n\t</script>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
var _ = templruntime.GeneratedTemplate
|
||||
@@ -0,0 +1,294 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
templ S3TablesTables(data dash.S3TablesTablesData) {
|
||||
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
|
||||
<h1 class="h2">
|
||||
<i class="fas fa-table me-2"></i>S3 Tables
|
||||
</h1>
|
||||
<div class="btn-toolbar mb-2 mb-md-0">
|
||||
<div class="btn-group me-2">
|
||||
<button type="button" class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#createS3TablesTableModal">
|
||||
<i class="fas fa-plus me-1"></i>Create Table
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
{{ bucketName, parseErr := s3tables.ParseBucketNameFromARN(data.BucketARN) }}
|
||||
if parseErr == nil {
|
||||
<a href={ templ.SafeURL(fmt.Sprintf("/object-store/s3tables/buckets/%s/namespaces", bucketName)) } class="btn btn-sm btn-outline-secondary">
|
||||
<i class="fas fa-arrow-left me-1"></i>Back to Namespaces
|
||||
</a>
|
||||
} else {
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary" disabled title="Invalid bucket ARN">
|
||||
<i class="fas fa-arrow-left me-1"></i>Back to Namespaces
|
||||
</button>
|
||||
}
|
||||
<span class="text-muted ms-2">Bucket ARN: { data.BucketARN }</span>
|
||||
<span class="text-muted ms-2">Namespace: { data.Namespace }</span>
|
||||
if parseErr != nil {
|
||||
<span class="text-danger ms-2">Invalid bucket ARN</span>
|
||||
}
|
||||
</div>
|
||||
<div id="s3tables-tables-content" data-bucket-arn={ data.BucketARN } data-namespace={ data.Namespace }>
|
||||
<div class="row mb-4">
|
||||
<div class="col-xl-4 col-md-6 mb-4">
|
||||
<div class="card border-left-primary shadow h-100 py-2">
|
||||
<div class="card-body">
|
||||
<div class="row no-gutters align-items-center">
|
||||
<div class="col mr-2">
|
||||
<div class="text-xs font-weight-bold text-primary text-uppercase mb-1">
|
||||
Total Tables
|
||||
</div>
|
||||
<div class="h5 mb-0 font-weight-bold text-gray-800">
|
||||
{ fmt.Sprintf("%d", data.TotalTables) }
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<i class="fas fa-table fa-2x text-gray-300"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-xl-4 col-md-6 mb-4">
|
||||
<div class="card border-left-info shadow h-100 py-2">
|
||||
<div class="card-body">
|
||||
<div class="row no-gutters align-items-center">
|
||||
<div class="col mr-2">
|
||||
<div class="text-xs font-weight-bold text-info text-uppercase mb-1">
|
||||
Last Updated
|
||||
</div>
|
||||
<div class="h6 mb-0 font-weight-bold text-gray-800">
|
||||
{ data.LastUpdated.Format("15:04") }
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-auto">
|
||||
<i class="fas fa-clock fa-2x text-gray-300"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="col-12">
|
||||
<div class="card shadow mb-4">
|
||||
<div class="card-header py-3 d-flex flex-row align-items-center justify-content-between">
|
||||
<h6 class="m-0 font-weight-bold text-primary">
|
||||
<i class="fas fa-table me-2"></i>Tables
|
||||
</h6>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="table-responsive">
|
||||
<table class="table table-hover" width="100%" cellspacing="0" id="s3tablesTablesTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Table ARN</th>
|
||||
<th>Created</th>
|
||||
<th>Modified</th>
|
||||
<th>Metadata</th>
|
||||
<th>Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
for _, table := range data.Tables {
|
||||
<tr>
|
||||
{{ tableName := table.Name }}
|
||||
<td>{ tableName }</td>
|
||||
<td class="text-muted small">{ table.TableARN }</td>
|
||||
<td>{ table.CreatedAt.Format("2006-01-02 15:04") }</td>
|
||||
<td>{ table.ModifiedAt.Format("2006-01-02 15:04") }</td>
|
||||
<td>
|
||||
if table.MetadataLocation != "" {
|
||||
<span class="text-muted small">{ table.MetadataLocation }</span>
|
||||
} else {
|
||||
<span class="text-muted">-</span>
|
||||
}
|
||||
</td>
|
||||
<td>
|
||||
<div class="btn-group btn-group-sm" role="group">
|
||||
<button type="button" class="btn btn-outline-success btn-sm s3tables-tags-btn" data-resource-arn={ table.TableARN } title="Tags">
|
||||
<i class="fas fa-tags"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-info btn-sm s3tables-table-policy-btn" data-table-arn={ table.TableARN } data-table-name={ tableName } title="Table Policy">
|
||||
<i class="fas fa-shield-alt"></i>
|
||||
</button>
|
||||
<button type="button" class="btn btn-outline-danger btn-sm s3tables-delete-table-btn" data-table-name={ tableName } title="Delete">
|
||||
<i class="fas fa-trash"></i>
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
if len(data.Tables) == 0 {
|
||||
<tr>
|
||||
<td colspan="6" class="text-center text-muted py-4">
|
||||
<i class="fas fa-table fa-3x mb-3 text-muted"></i>
|
||||
<div>
|
||||
<h5>No tables found</h5>
|
||||
<p>Create your first table to start storing data.</p>
|
||||
<button type="button" class="btn btn-primary" data-bs-toggle="modal" data-bs-target="#createS3TablesTableModal">
|
||||
<i class="fas fa-plus me-1"></i>Create Table
|
||||
</button>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="createS3TablesTableModal" tabindex="-1" aria-labelledby="createS3TablesTableModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="createS3TablesTableModalLabel">
|
||||
<i class="fas fa-plus me-2"></i>Create Table
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form id="createS3TablesTableForm">
|
||||
<div class="modal-body">
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesTableName" class="form-label">Table Name</label>
|
||||
<input type="text" class="form-control" id="s3tablesTableName" name="name" required/>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesTableFormat" class="form-label">Format</label>
|
||||
<select class="form-select" id="s3tablesTableFormat" name="format">
|
||||
<option value="ICEBERG" selected>ICEBERG</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesTableMetadata" class="form-label">Metadata JSON (optional)</label>
|
||||
<textarea class="form-control" id="s3tablesTableMetadata" name="metadata" rows="6" placeholder="{ }"></textarea>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesTableTags" class="form-label">Tags</label>
|
||||
<input type="text" class="form-control" id="s3tablesTableTags" name="tags" placeholder="key1=value1,key2=value2"/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<i class="fas fa-plus me-1"></i>Create
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="deleteS3TablesTableModal" tabindex="-1" aria-labelledby="deleteS3TablesTableModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="deleteS3TablesTableModalLabel">
|
||||
<i class="fas fa-exclamation-triangle me-2 text-warning"></i>Delete Table
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<p>Are you sure you want to delete the table <strong id="deleteS3TablesTableName"></strong>?</p>
|
||||
<div class="mb-3">
|
||||
<label for="deleteS3TablesTableVersion" class="form-label">Version Token (optional)</label>
|
||||
<input type="text" class="form-control" id="deleteS3TablesTableVersion" placeholder="Version token"/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="button" class="btn btn-danger" onclick="deleteS3TablesTable()">
|
||||
<i class="fas fa-trash me-1"></i>Delete
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="s3tablesTablePolicyModal" tabindex="-1" aria-labelledby="s3tablesTablePolicyModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="s3tablesTablePolicyModalLabel">
|
||||
<i class="fas fa-shield-alt me-2"></i>Table Policy
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form id="s3tablesTablePolicyForm">
|
||||
<div class="modal-body">
|
||||
<input type="hidden" id="s3tablesTablePolicyBucketArn" name="bucket_arn"/>
|
||||
<input type="hidden" id="s3tablesTablePolicyNamespace" name="namespace"/>
|
||||
<input type="hidden" id="s3tablesTablePolicyName" name="name"/>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesTablePolicyText" class="form-label">Policy JSON</label>
|
||||
<textarea class="form-control" id="s3tablesTablePolicyText" name="policy" rows="12" placeholder="{ }"></textarea>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
|
||||
<button type="button" class="btn btn-outline-danger" onclick="deleteS3TablesTablePolicy()">
|
||||
<i class="fas fa-trash me-1"></i>Delete Policy
|
||||
</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<i class="fas fa-save me-1"></i>Save Policy
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal fade" id="s3tablesTagsModal" tabindex="-1" aria-labelledby="s3tablesTagsModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog modal-lg">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="s3tablesTagsModalLabel">
|
||||
<i class="fas fa-tags me-2"></i>Resource Tags
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form id="s3tablesTagsForm">
|
||||
<div class="modal-body">
|
||||
<input type="hidden" id="s3tablesTagsResourceArn" name="resource_arn"/>
|
||||
<div class="mb-3">
|
||||
<label class="form-label">Existing Tags</label>
|
||||
<pre class="bg-light p-3 border rounded" id="s3tablesTagsList">Loading...</pre>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesTagsInput" class="form-label">Add or Update Tags</label>
|
||||
<input type="text" class="form-control" id="s3tablesTagsInput" placeholder="key1=value1,key2=value2"/>
|
||||
</div>
|
||||
<div class="mb-3">
|
||||
<label for="s3tablesTagsDeleteInput" class="form-label">Remove Tag Keys</label>
|
||||
<input type="text" class="form-control" id="s3tablesTagsDeleteInput" placeholder="key1,key2"/>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
|
||||
<button type="button" class="btn btn-outline-danger" onclick="deleteS3TablesTags()">
|
||||
<i class="fas fa-trash me-1"></i>Remove Tags
|
||||
</button>
|
||||
<button type="submit" class="btn btn-primary">
|
||||
<i class="fas fa-save me-1"></i>Update Tags
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
initS3TablesTables();
|
||||
});
|
||||
</script>
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -163,6 +163,11 @@ templ Layout(c *gin.Context, content templ.Component) {
|
||||
<i class="fas fa-cube me-2"></i>Buckets
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link py-2" href="/object-store/s3tables/buckets">
|
||||
<i class="fas fa-table me-2"></i>Table Buckets
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link py-2" href="/object-store/users">
|
||||
<i class="fas fa-users me-2"></i>Users
|
||||
@@ -362,6 +367,7 @@ templ Layout(c *gin.Context, content templ.Component) {
|
||||
<!-- Custom JS -->
|
||||
<script src="/static/js/admin.js"></script>
|
||||
<script src="/static/js/iam-utils.js"></script>
|
||||
<script src="/static/js/s3tables.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
}
|
||||
@@ -430,4 +436,4 @@ templ LoginForm(c *gin.Context, title string, errorMessage string) {
|
||||
<script src="/static/js/bootstrap.bundle.min.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "\" id=\"storageSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/storage/volumes\"><i class=\"fas fa-database me-2\"></i>Volumes</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/storage/ec-shards\"><i class=\"fas fa-th-large me-2\"></i>EC Volumes</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/storage/collections\"><i class=\"fas fa-layer-group me-2\"></i>Collections</a></li></ul></div></li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MANAGEMENT</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"/files\"><i class=\"fas fa-folder me-2\"></i>File Browser</a></li><li class=\"nav-item\"><a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#objectStoreSubmenu\" aria-expanded=\"false\" aria-controls=\"objectStoreSubmenu\"><i class=\"fas fa-cloud me-2\"></i>Object Store <i class=\"fas fa-chevron-down ms-auto\"></i></a><div class=\"collapse\" id=\"objectStoreSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/buckets\"><i class=\"fas fa-cube me-2\"></i>Buckets</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/users\"><i class=\"fas fa-users me-2\"></i>Users</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/service-accounts\"><i class=\"fas fa-robot me-2\"></i>Service Accounts</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/policies\"><i class=\"fas fa-shield-alt me-2\"></i>Policies</a></li></ul></div></li><li class=\"nav-item\">")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "\" id=\"storageSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/storage/volumes\"><i class=\"fas fa-database me-2\"></i>Volumes</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/storage/ec-shards\"><i class=\"fas fa-th-large me-2\"></i>EC Volumes</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/storage/collections\"><i class=\"fas fa-layer-group me-2\"></i>Collections</a></li></ul></div></li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MANAGEMENT</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"/files\"><i class=\"fas fa-folder me-2\"></i>File Browser</a></li><li class=\"nav-item\"><a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#objectStoreSubmenu\" aria-expanded=\"false\" aria-controls=\"objectStoreSubmenu\"><i class=\"fas fa-cloud me-2\"></i>Object Store <i class=\"fas fa-chevron-down ms-auto\"></i></a><div class=\"collapse\" id=\"objectStoreSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/buckets\"><i class=\"fas fa-cube me-2\"></i>Buckets</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/s3tables/buckets\"><i class=\"fas fa-table me-2\"></i>Table Buckets</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/users\"><i class=\"fas fa-users me-2\"></i>Users</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/service-accounts\"><i class=\"fas fa-robot me-2\"></i>Service Accounts</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/policies\"><i class=\"fas fa-shield-alt me-2\"></i>Policies</a></li></ul></div></li><li class=\"nav-item\">")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -271,7 +271,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
|
||||
var templ_7745c5c3_Var13 templ.SafeURL
|
||||
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 282, Col: 117}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 287, Col: 117}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -306,7 +306,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
|
||||
var templ_7745c5c3_Var16 string
|
||||
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 283, Col: 109}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 288, Col: 109}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -324,7 +324,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
|
||||
var templ_7745c5c3_Var17 templ.SafeURL
|
||||
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 286, Col: 110}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 291, Col: 110}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -359,7 +359,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
|
||||
var templ_7745c5c3_Var20 string
|
||||
templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 287, Col: 109}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 292, Col: 109}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -392,7 +392,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
|
||||
var templ_7745c5c3_Var21 templ.SafeURL
|
||||
templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 299, Col: 106}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 304, Col: 106}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -427,7 +427,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
|
||||
var templ_7745c5c3_Var24 string
|
||||
templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 300, Col: 105}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 305, Col: 105}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -488,7 +488,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
|
||||
var templ_7745c5c3_Var25 string
|
||||
templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 347, Col: 60}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 352, Col: 60}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -501,7 +501,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
|
||||
var templ_7745c5c3_Var26 string
|
||||
templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 347, Col: 102}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 352, Col: 102}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -517,7 +517,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
}
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "</small></div></footer><!-- Bootstrap JS --><script src=\"/static/js/bootstrap.bundle.min.js\"></script><!-- Modal Alerts JS (replaces native alert/confirm) --><script src=\"/static/js/modal-alerts.js\"></script><!-- Custom JS --><script src=\"/static/js/admin.js\"></script><script src=\"/static/js/iam-utils.js\"></script></body></html>")
|
||||
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "</small></div></footer><!-- Bootstrap JS --><script src=\"/static/js/bootstrap.bundle.min.js\"></script><!-- Modal Alerts JS (replaces native alert/confirm) --><script src=\"/static/js/modal-alerts.js\"></script><!-- Custom JS --><script src=\"/static/js/admin.js\"></script><script src=\"/static/js/iam-utils.js\"></script><script src=\"/static/js/s3tables.js\"></script></body></html>")
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ_7745c5c3_Err
|
||||
}
|
||||
@@ -553,7 +553,7 @@ func LoginForm(c *gin.Context, title string, errorMessage string) templ.Componen
|
||||
var templ_7745c5c3_Var28 string
|
||||
templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(title)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 374, Col: 17}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 380, Col: 17}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -566,7 +566,7 @@ func LoginForm(c *gin.Context, title string, errorMessage string) templ.Componen
|
||||
var templ_7745c5c3_Var29 string
|
||||
templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(title)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 388, Col: 57}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 394, Col: 57}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
@@ -584,7 +584,7 @@ func LoginForm(c *gin.Context, title string, errorMessage string) templ.Componen
|
||||
var templ_7745c5c3_Var30 string
|
||||
templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
|
||||
if templ_7745c5c3_Err != nil {
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 395, Col: 45}
|
||||
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 401, Col: 45}
|
||||
}
|
||||
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
|
||||
if templ_7745c5c3_Err != nil {
|
||||
|
||||
+11
-11
@@ -32,17 +32,17 @@ func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *
|
||||
}
|
||||
|
||||
type LiveLock struct {
|
||||
key string
|
||||
renewToken string
|
||||
expireAtNs int64
|
||||
hostFiler pb.ServerAddress
|
||||
cancelCh chan struct{}
|
||||
grpcDialOption grpc.DialOption
|
||||
isLocked int32 // 0 = unlocked, 1 = locked; use atomic operations
|
||||
self string
|
||||
lc *LockClient
|
||||
owner string
|
||||
lockTTL time.Duration
|
||||
key string
|
||||
renewToken string
|
||||
expireAtNs int64
|
||||
hostFiler pb.ServerAddress
|
||||
cancelCh chan struct{}
|
||||
grpcDialOption grpc.DialOption
|
||||
isLocked int32 // 0 = unlocked, 1 = locked; use atomic operations
|
||||
self string
|
||||
lc *LockClient
|
||||
owner string
|
||||
lockTTL time.Duration
|
||||
consecutiveFailures int // Track connection failures to trigger fallback
|
||||
}
|
||||
|
||||
|
||||
+24
-6
@@ -477,23 +477,41 @@ func (fo *FilerOptions) startFiler() {
|
||||
if filerLocalListener != nil {
|
||||
go func() {
|
||||
if err := newHttpServer(defaultMux, tlsConfig).ServeTLS(filerLocalListener, "", ""); err != nil {
|
||||
glog.Errorf("Filer Fail to serve: %v", e)
|
||||
glog.Errorf("Filer Fail to serve: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err := newHttpServer(defaultMux, tlsConfig).ServeTLS(filerListener, "", ""); err != nil {
|
||||
glog.Fatalf("Filer Fail to serve: %v", e)
|
||||
httpS := newHttpServer(defaultMux, tlsConfig)
|
||||
if MiniClusterCtx != nil {
|
||||
ctx := MiniClusterCtx
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
httpS.Shutdown(context.Background())
|
||||
grpcS.Stop()
|
||||
}()
|
||||
}
|
||||
if err := httpS.ServeTLS(filerListener, "", ""); err != nil && err != http.ErrServerClosed {
|
||||
glog.Fatalf("Filer Fail to serve: %v", err)
|
||||
}
|
||||
} else {
|
||||
if filerLocalListener != nil {
|
||||
go func() {
|
||||
if err := newHttpServer(defaultMux, nil).Serve(filerLocalListener); err != nil {
|
||||
glog.Errorf("Filer Fail to serve: %v", e)
|
||||
glog.Errorf("Filer Fail to serve: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err := newHttpServer(defaultMux, nil).Serve(filerListener); err != nil {
|
||||
glog.Fatalf("Filer Fail to serve: %v", e)
|
||||
httpS := newHttpServer(defaultMux, nil)
|
||||
if MiniClusterCtx != nil {
|
||||
ctx := MiniClusterCtx
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
httpS.Shutdown(context.Background())
|
||||
grpcS.Stop()
|
||||
}()
|
||||
}
|
||||
if err := httpS.Serve(filerListener); err != nil && err != http.ErrServerClosed {
|
||||
glog.Fatalf("Filer Fail to serve: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//go:build !linux && !darwin
|
||||
// +build !linux,!darwin
|
||||
//go:build !darwin && !freebsd && !linux
|
||||
// +build !darwin,!freebsd,!linux
|
||||
|
||||
package command
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//go:build linux || darwin
|
||||
// +build linux darwin
|
||||
//go:build darwin || freebsd || linux
|
||||
// +build darwin freebsd linux
|
||||
|
||||
package command
|
||||
|
||||
@@ -157,6 +157,13 @@ func runFuse(cmd *Command, args []string) bool {
|
||||
} else {
|
||||
panic(fmt.Errorf("cacheMetaTtlSec: %s", err))
|
||||
}
|
||||
case "dirIdleEvictSec":
|
||||
if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
|
||||
intValue := int(parsed)
|
||||
mountOptions.dirIdleEvictSec = &intValue
|
||||
} else {
|
||||
panic(fmt.Errorf("dirIdleEvictSec: %s", err))
|
||||
}
|
||||
case "concurrentWriters":
|
||||
i++
|
||||
if parsed, err := strconv.ParseInt(parameter.value, 0, 32); err == nil {
|
||||
|
||||
@@ -311,7 +311,14 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
ms.Topo.HashicorpRaft.LeadershipTransfer()
|
||||
}
|
||||
})
|
||||
select {}
|
||||
ctx := MiniClusterCtx
|
||||
if ctx != nil {
|
||||
<-ctx.Done()
|
||||
ms.Shutdown()
|
||||
grpcS.Stop()
|
||||
} else {
|
||||
select {}
|
||||
}
|
||||
}
|
||||
|
||||
func isSingleMasterMode(peers string) bool {
|
||||
|
||||
+20
-2
@@ -59,6 +59,8 @@ var (
|
||||
miniEnableS3 *bool
|
||||
miniEnableAdminUI *bool
|
||||
miniS3IamReadOnly *bool
|
||||
// MiniClusterCtx is the context for the mini cluster. If set, the mini cluster will stop when the context is cancelled.
|
||||
MiniClusterCtx context.Context
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -821,7 +823,12 @@ func runMini(cmd *Command, args []string) bool {
|
||||
// Save configuration to file for persistence and documentation
|
||||
saveMiniConfiguration(*miniDataFolders)
|
||||
|
||||
select {}
|
||||
if MiniClusterCtx != nil {
|
||||
<-MiniClusterCtx.Done()
|
||||
} else {
|
||||
select {}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// startMiniServices starts all mini services with proper dependency coordination
|
||||
@@ -928,7 +935,12 @@ func startS3Service() {
|
||||
func startMiniAdminWithWorker(allServicesReady chan struct{}) {
|
||||
defer close(allServicesReady) // Ensure channel is always closed on all paths
|
||||
|
||||
ctx := context.Background()
|
||||
var ctx context.Context
|
||||
if MiniClusterCtx != nil {
|
||||
ctx = MiniClusterCtx
|
||||
} else {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
// Determine bind IP for health checks
|
||||
bindIp := getBindIp()
|
||||
@@ -1101,6 +1113,12 @@ func startMiniWorker() {
|
||||
// Metrics server is already started in the main init function above, so no need to start it again here
|
||||
|
||||
// Start the worker
|
||||
if MiniClusterCtx != nil {
|
||||
go func() {
|
||||
<-MiniClusterCtx.Done()
|
||||
workerInstance.Stop()
|
||||
}()
|
||||
}
|
||||
err = workerInstance.Start()
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to start worker: %v", err)
|
||||
|
||||
@@ -48,6 +48,8 @@ type MountOptions struct {
|
||||
rdmaMaxConcurrent *int
|
||||
rdmaTimeoutMs *int
|
||||
|
||||
dirIdleEvictSec *int
|
||||
|
||||
// FUSE performance options
|
||||
writebackCache *bool
|
||||
asyncDio *bool
|
||||
@@ -107,6 +109,8 @@ func init() {
|
||||
mountOptions.rdmaMaxConcurrent = cmdMount.Flag.Int("rdma.maxConcurrent", 64, "max concurrent RDMA operations")
|
||||
mountOptions.rdmaTimeoutMs = cmdMount.Flag.Int("rdma.timeoutMs", 5000, "RDMA operation timeout in milliseconds")
|
||||
|
||||
mountOptions.dirIdleEvictSec = cmdMount.Flag.Int("dirIdleEvictSec", 600, "seconds to evict idle cached directories (0 to disable)")
|
||||
|
||||
mountCpuProfile = cmdMount.Flag.String("cpuprofile", "", "cpu profile output file")
|
||||
mountMemProfile = cmdMount.Flag.String("memprofile", "", "memory profile output file")
|
||||
mountReadRetryTime = cmdMount.Flag.Duration("readRetryTime", 6*time.Second, "maximum read retry wait time")
|
||||
|
||||
@@ -270,6 +270,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
RdmaReadOnly: *option.rdmaReadOnly,
|
||||
RdmaMaxConcurrent: *option.rdmaMaxConcurrent,
|
||||
RdmaTimeoutMs: *option.rdmaTimeoutMs,
|
||||
DirIdleEvictSec: *option.dirIdleEvictSec,
|
||||
})
|
||||
|
||||
// create mount root
|
||||
|
||||
+20
-2
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
@@ -405,7 +406,16 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err = newHttpServer(router, tlsConfig).ServeTLS(s3ApiListener, "", ""); err != nil {
|
||||
httpS := newHttpServer(router, tlsConfig)
|
||||
if MiniClusterCtx != nil {
|
||||
ctx := MiniClusterCtx
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
httpS.Shutdown(context.Background())
|
||||
grpcS.Stop()
|
||||
}()
|
||||
}
|
||||
if err = httpS.ServeTLS(s3ApiListener, "", ""); err != nil && err != http.ErrServerClosed {
|
||||
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
||||
}
|
||||
} else {
|
||||
@@ -438,7 +448,15 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err = newHttpServer(router, nil).Serve(s3ApiListener); err != nil {
|
||||
httpS := newHttpServer(router, nil)
|
||||
if MiniClusterCtx != nil {
|
||||
go func() {
|
||||
<-MiniClusterCtx.Done()
|
||||
httpS.Shutdown(context.Background())
|
||||
grpcS.Stop()
|
||||
}()
|
||||
}
|
||||
if err = httpS.Serve(s3ApiListener); err != nil && err != http.ErrServerClosed {
|
||||
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,7 +50,7 @@ expires_after_seconds = 10 # seconds
|
||||
|
||||
# If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
|
||||
# - f.e. the S3 API Shim generates the JWT
|
||||
# - the Filer server validates the JWT on writing
|
||||
# - the Filer server validates the JWT on reading
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
[jwt.filer_signing.read]
|
||||
key = ""
|
||||
|
||||
+11
-2
@@ -319,8 +319,17 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
||||
stopChan <- true
|
||||
})
|
||||
|
||||
select {
|
||||
case <-stopChan:
|
||||
ctx := MiniClusterCtx
|
||||
if ctx != nil {
|
||||
select {
|
||||
case <-stopChan:
|
||||
case <-ctx.Done():
|
||||
shutdown(publicHttpDown, clusterHttpServer, grpcS, volumeServer)
|
||||
}
|
||||
} else {
|
||||
select {
|
||||
case <-stopChan:
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -137,14 +137,21 @@ func (wo *WebDavOption) startWebDav() bool {
|
||||
glog.Fatalf("WebDav Server listener on %s error: %v", listenAddress, err)
|
||||
}
|
||||
|
||||
if MiniClusterCtx != nil {
|
||||
go func() {
|
||||
<-MiniClusterCtx.Done()
|
||||
httpS.Shutdown(context.Background())
|
||||
}()
|
||||
}
|
||||
|
||||
if *wo.tlsPrivateKey != "" {
|
||||
glog.V(0).Infof("Start Seaweed WebDav Server %s at https %s", version.Version(), listenAddress)
|
||||
if err = httpS.ServeTLS(webDavListener, *wo.tlsCertificate, *wo.tlsPrivateKey); err != nil {
|
||||
if err = httpS.ServeTLS(webDavListener, *wo.tlsCertificate, *wo.tlsPrivateKey); err != nil && err != http.ErrServerClosed {
|
||||
glog.Fatalf("WebDav Server Fail to serve: %v", err)
|
||||
}
|
||||
} else {
|
||||
glog.V(0).Infof("Start Seaweed WebDav Server %s at http %s", version.Version(), listenAddress)
|
||||
if err = httpS.Serve(webDavListener); err != nil {
|
||||
if err = httpS.Serve(webDavListener); err != nil && err != http.ErrServerClosed {
|
||||
glog.Fatalf("WebDav Server Fail to serve: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -427,7 +427,7 @@ func (store *FilerEtcStore) DeleteAccessKey(ctx context.Context, username string
|
||||
|
||||
func (store *FilerEtcStore) saveIdentity(ctx context.Context, identity *iam_pb.Identity) error {
|
||||
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := json.Marshal(identity)
|
||||
data, err := json.MarshalIndent(identity, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -67,7 +67,7 @@ func (store *FilerEtcStore) saveServiceAccount(ctx context.Context, sa *iam_pb.S
|
||||
return err
|
||||
}
|
||||
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := json.Marshal(sa)
|
||||
data, err := json.MarshalIndent(sa, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -397,7 +397,7 @@ func (p *OIDCProvider) ValidateToken(ctx context.Context, token string) (*provid
|
||||
validatedToken, err := jwt.ParseWithClaims(token, claims, func(token *jwt.Token) (interface{}, error) {
|
||||
// Verify signing method
|
||||
switch token.Method.(type) {
|
||||
case *jwt.SigningMethodRSA:
|
||||
case *jwt.SigningMethodRSA, *jwt.SigningMethodECDSA:
|
||||
return publicKey, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported signing method: %v", token.Header["alg"])
|
||||
|
||||
@@ -2,6 +2,8 @@ package oidc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"encoding/base64"
|
||||
@@ -188,7 +190,123 @@ func TestOIDCProviderJWTValidation(t *testing.T) {
|
||||
})
|
||||
|
||||
_, err := provider.ValidateToken(context.Background(), token)
|
||||
assert.Error(t, err)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, providers.ErrProviderInvalidToken)
|
||||
})
|
||||
}
|
||||
|
||||
func TestOIDCProviderJWTValidationECDSA(t *testing.T) {
|
||||
privateKey, publicKey := generateTestECKeys(t)
|
||||
x, y := encodeECPublicKey(t, publicKey)
|
||||
|
||||
jwks := map[string]interface{}{
|
||||
"keys": []map[string]interface{}{
|
||||
{
|
||||
"kty": "EC",
|
||||
"kid": "test-ec-key-id",
|
||||
"use": "sig",
|
||||
"alg": "ES256",
|
||||
"crv": "P-256",
|
||||
"x": x,
|
||||
"y": y,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/.well-known/openid_configuration" {
|
||||
config := map[string]interface{}{
|
||||
"issuer": "http://" + r.Host,
|
||||
"jwks_uri": "http://" + r.Host + "/jwks",
|
||||
}
|
||||
json.NewEncoder(w).Encode(config)
|
||||
} else if r.URL.Path == "/jwks" {
|
||||
json.NewEncoder(w).Encode(jwks)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
provider := NewOIDCProvider("test-oidc-ecdsa")
|
||||
config := &OIDCConfig{
|
||||
Issuer: server.URL,
|
||||
ClientID: "test-client",
|
||||
JWKSUri: server.URL + "/jwks",
|
||||
}
|
||||
|
||||
err := provider.Initialize(config)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("valid token", func(t *testing.T) {
|
||||
token := createTestECDSAJWT(t, privateKey, jwt.MapClaims{
|
||||
"iss": server.URL,
|
||||
"aud": "test-client",
|
||||
"sub": "user789",
|
||||
"exp": time.Now().Add(time.Hour).Unix(),
|
||||
"iat": time.Now().Unix(),
|
||||
})
|
||||
|
||||
claims, err := provider.ValidateToken(context.Background(), token)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, claims)
|
||||
assert.Equal(t, "user789", claims.Subject)
|
||||
assert.Equal(t, server.URL, claims.Issuer)
|
||||
})
|
||||
|
||||
t.Run("expired token", func(t *testing.T) {
|
||||
token := createTestECDSAJWT(t, privateKey, jwt.MapClaims{
|
||||
"iss": server.URL,
|
||||
"aud": "test-client",
|
||||
"sub": "user789",
|
||||
"exp": time.Now().Add(-time.Hour).Unix(),
|
||||
"iat": time.Now().Add(-time.Hour * 2).Unix(),
|
||||
})
|
||||
|
||||
_, err := provider.ValidateToken(context.Background(), token)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, providers.ErrProviderTokenExpired)
|
||||
})
|
||||
|
||||
t.Run("invalid signature", func(t *testing.T) {
|
||||
wrongKey, _ := generateTestECKeys(t)
|
||||
token := createTestECDSAJWT(t, wrongKey, jwt.MapClaims{
|
||||
"iss": server.URL,
|
||||
"aud": "test-client",
|
||||
"sub": "user789",
|
||||
"exp": time.Now().Add(time.Hour).Unix(),
|
||||
"iat": time.Now().Unix(),
|
||||
})
|
||||
|
||||
_, err := provider.ValidateToken(context.Background(), token)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, providers.ErrProviderInvalidToken)
|
||||
})
|
||||
|
||||
t.Run("invalid issuer", func(t *testing.T) {
|
||||
token := createTestECDSAJWT(t, privateKey, jwt.MapClaims{
|
||||
"iss": "http://wrong-issuer",
|
||||
"aud": "test-client",
|
||||
"sub": "user789",
|
||||
"exp": time.Now().Add(time.Hour).Unix(),
|
||||
"iat": time.Now().Unix(),
|
||||
})
|
||||
|
||||
_, err := provider.ValidateToken(context.Background(), token)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, providers.ErrProviderInvalidIssuer)
|
||||
})
|
||||
|
||||
t.Run("invalid audience", func(t *testing.T) {
|
||||
token := createTestECDSAJWT(t, privateKey, jwt.MapClaims{
|
||||
"iss": server.URL,
|
||||
"aud": "wrong-client",
|
||||
"sub": "user789",
|
||||
"exp": time.Now().Add(time.Hour).Unix(),
|
||||
"iat": time.Now().Unix(),
|
||||
})
|
||||
|
||||
_, err := provider.ValidateToken(context.Background(), token)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, providers.ErrProviderInvalidAudience)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -435,6 +553,12 @@ func generateTestKeys(t *testing.T) (*rsa.PrivateKey, *rsa.PublicKey) {
|
||||
return privateKey, &privateKey.PublicKey
|
||||
}
|
||||
|
||||
func generateTestECKeys(t *testing.T) (*ecdsa.PrivateKey, *ecdsa.PublicKey) {
|
||||
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
require.NoError(t, err)
|
||||
return privateKey, &privateKey.PublicKey
|
||||
}
|
||||
|
||||
func createTestJWT(t *testing.T, privateKey *rsa.PrivateKey, claims jwt.MapClaims) string {
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)
|
||||
token.Header["kid"] = "test-key-id"
|
||||
@@ -444,11 +568,35 @@ func createTestJWT(t *testing.T, privateKey *rsa.PrivateKey, claims jwt.MapClaim
|
||||
return tokenString
|
||||
}
|
||||
|
||||
func createTestECDSAJWT(t *testing.T, privateKey *ecdsa.PrivateKey, claims jwt.MapClaims) string {
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodES256, claims)
|
||||
token.Header["kid"] = "test-ec-key-id"
|
||||
|
||||
tokenString, err := token.SignedString(privateKey)
|
||||
require.NoError(t, err)
|
||||
return tokenString
|
||||
}
|
||||
|
||||
func encodePublicKey(t *testing.T, publicKey *rsa.PublicKey) string {
|
||||
// Properly encode the RSA modulus (N) as base64url
|
||||
return base64.RawURLEncoding.EncodeToString(publicKey.N.Bytes())
|
||||
}
|
||||
|
||||
func encodeECPublicKey(t *testing.T, publicKey *ecdsa.PublicKey) (string, string) {
|
||||
// RFC 7518 §6.2.1.2 requires EC coordinates to be zero-padded to the full field size
|
||||
curveParams := publicKey.Curve.Params()
|
||||
size := (curveParams.BitSize + 7) / 8
|
||||
xBytes := publicKey.X.Bytes()
|
||||
yBytes := publicKey.Y.Bytes()
|
||||
xPadded := make([]byte, size)
|
||||
yPadded := make([]byte, size)
|
||||
// Right-align the coordinate bytes and leave leading zeros for padding
|
||||
copy(xPadded[size-len(xBytes):], xBytes)
|
||||
copy(yPadded[size-len(yBytes):], yBytes)
|
||||
return base64.RawURLEncoding.EncodeToString(xPadded),
|
||||
base64.RawURLEncoding.EncodeToString(yPadded)
|
||||
}
|
||||
|
||||
func setupOIDCTestServer(t *testing.T, publicKey *rsa.PublicKey) *httptest.Server {
|
||||
jwks := map[string]interface{}{
|
||||
"keys": []map[string]interface{}{
|
||||
|
||||
+158
-9
@@ -22,6 +22,19 @@ type InodeEntry struct {
|
||||
isDirectory bool
|
||||
isChildrenCached bool
|
||||
cachedExpiresTime time.Time
|
||||
lastAccess time.Time
|
||||
lastRefresh time.Time
|
||||
updateWindowStart time.Time
|
||||
updateCount int
|
||||
needsRefresh bool
|
||||
}
|
||||
|
||||
func (ie *InodeEntry) resetCacheState() {
|
||||
ie.isChildrenCached = false
|
||||
ie.cachedExpiresTime = time.Time{}
|
||||
ie.needsRefresh = false
|
||||
ie.updateCount = 0
|
||||
ie.updateWindowStart = time.Time{}
|
||||
}
|
||||
|
||||
func (ie *InodeEntry) removeOnePath(p util.FullPath) bool {
|
||||
@@ -51,7 +64,12 @@ func NewInodeToPath(root util.FullPath, ttlSec int) *InodeToPath {
|
||||
path2inode: make(map[util.FullPath]uint64),
|
||||
cacheMetaTtlSec: time.Second * time.Duration(ttlSec),
|
||||
}
|
||||
t.inode2path[1] = &InodeEntry{[]util.FullPath{root}, 1, true, false, time.Time{}}
|
||||
t.inode2path[1] = &InodeEntry{
|
||||
paths: []util.FullPath{root},
|
||||
nlookup: 1,
|
||||
isDirectory: true,
|
||||
lastAccess: time.Now(),
|
||||
}
|
||||
t.path2inode[root] = 1
|
||||
|
||||
return t
|
||||
@@ -94,9 +112,16 @@ func (i *InodeToPath) Lookup(path util.FullPath, unixTime int64, isDirectory boo
|
||||
}
|
||||
} else {
|
||||
if !isLookup {
|
||||
i.inode2path[inode] = &InodeEntry{[]util.FullPath{path}, 0, isDirectory, false, time.Time{}}
|
||||
i.inode2path[inode] = &InodeEntry{
|
||||
paths: []util.FullPath{path},
|
||||
isDirectory: isDirectory,
|
||||
}
|
||||
} else {
|
||||
i.inode2path[inode] = &InodeEntry{[]util.FullPath{path}, 1, isDirectory, false, time.Time{}}
|
||||
i.inode2path[inode] = &InodeEntry{
|
||||
paths: []util.FullPath{path},
|
||||
nlookup: 1,
|
||||
isDirectory: isDirectory,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,8 +188,14 @@ func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) {
|
||||
return
|
||||
}
|
||||
path.isChildrenCached = true
|
||||
now := time.Now()
|
||||
path.lastAccess = now
|
||||
path.lastRefresh = now
|
||||
path.updateCount = 0
|
||||
path.needsRefresh = false
|
||||
path.updateWindowStart = time.Time{}
|
||||
if i.cacheMetaTtlSec > 0 {
|
||||
path.cachedExpiresTime = time.Now().Add(i.cacheMetaTtlSec)
|
||||
path.cachedExpiresTime = now.Add(i.cacheMetaTtlSec)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -195,6 +226,125 @@ func (i *InodeToPath) HasInode(inode uint64) bool {
|
||||
return found
|
||||
}
|
||||
|
||||
func (i *InodeToPath) InvalidateAllChildrenCache() {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
for _, entry := range i.inode2path {
|
||||
if entry.isDirectory && entry.isChildrenCached {
|
||||
entry.resetCacheState()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *InodeToPath) InvalidateChildrenCache(fullpath util.FullPath) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
inode, found := i.path2inode[fullpath]
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
entry, found := i.inode2path[inode]
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
entry.resetCacheState()
|
||||
}
|
||||
|
||||
func (i *InodeToPath) TouchDirectory(fullpath util.FullPath) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
inode, found := i.path2inode[fullpath]
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
entry, found := i.inode2path[inode]
|
||||
if !found || !entry.isDirectory {
|
||||
return
|
||||
}
|
||||
entry.lastAccess = time.Now()
|
||||
}
|
||||
|
||||
func (i *InodeToPath) RecordDirectoryUpdate(fullpath util.FullPath, now time.Time, window time.Duration, threshold int) bool {
|
||||
if threshold <= 0 || window <= 0 {
|
||||
return false
|
||||
}
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
inode, found := i.path2inode[fullpath]
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
entry, found := i.inode2path[inode]
|
||||
if !found || !entry.isDirectory || !entry.isChildrenCached {
|
||||
return false
|
||||
}
|
||||
if entry.updateWindowStart.IsZero() || now.Sub(entry.updateWindowStart) > window {
|
||||
entry.updateWindowStart = now
|
||||
entry.updateCount = 0
|
||||
}
|
||||
entry.updateCount++
|
||||
if entry.updateCount >= threshold {
|
||||
entry.needsRefresh = true
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (i *InodeToPath) NeedsRefresh(fullpath util.FullPath) bool {
|
||||
i.RLock()
|
||||
defer i.RUnlock()
|
||||
inode, found := i.path2inode[fullpath]
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
entry, found := i.inode2path[inode]
|
||||
if !found || !entry.isDirectory {
|
||||
return false
|
||||
}
|
||||
return entry.isChildrenCached && entry.needsRefresh
|
||||
}
|
||||
|
||||
func (i *InodeToPath) MarkDirectoryRefreshed(fullpath util.FullPath, now time.Time) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
inode, found := i.path2inode[fullpath]
|
||||
if !found {
|
||||
return
|
||||
}
|
||||
entry, found := i.inode2path[inode]
|
||||
if !found || !entry.isDirectory {
|
||||
return
|
||||
}
|
||||
entry.lastRefresh = now
|
||||
entry.lastAccess = now
|
||||
entry.updateCount = 0
|
||||
entry.needsRefresh = false
|
||||
entry.updateWindowStart = time.Time{}
|
||||
if i.cacheMetaTtlSec > 0 {
|
||||
entry.cachedExpiresTime = now.Add(i.cacheMetaTtlSec)
|
||||
}
|
||||
}
|
||||
|
||||
func (i *InodeToPath) CollectEvictableDirs(now time.Time, idle time.Duration) []util.FullPath {
|
||||
if idle <= 0 {
|
||||
return nil
|
||||
}
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
var dirs []util.FullPath
|
||||
for _, entry := range i.inode2path {
|
||||
if !entry.isDirectory || !entry.isChildrenCached {
|
||||
continue
|
||||
}
|
||||
if entry.lastAccess.IsZero() || now.Sub(entry.lastAccess) < idle {
|
||||
continue
|
||||
}
|
||||
entry.resetCacheState()
|
||||
dirs = append(dirs, entry.paths...)
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
|
||||
func (i *InodeToPath) AddPath(inode uint64, path util.FullPath) {
|
||||
i.Lock()
|
||||
defer i.Unlock()
|
||||
@@ -206,10 +356,9 @@ func (i *InodeToPath) AddPath(inode uint64, path util.FullPath) {
|
||||
ie.nlookup++
|
||||
} else {
|
||||
i.inode2path[inode] = &InodeEntry{
|
||||
paths: []util.FullPath{path},
|
||||
nlookup: 1,
|
||||
isDirectory: false,
|
||||
isChildrenCached: false,
|
||||
paths: []util.FullPath{path},
|
||||
nlookup: 1,
|
||||
isDirectory: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -257,7 +406,7 @@ func (i *InodeToPath) MovePath(sourcePath, targetPath util.FullPath) (sourceInod
|
||||
entry.paths[i] = targetPath
|
||||
}
|
||||
}
|
||||
entry.isChildrenCached = false
|
||||
entry.resetCacheState()
|
||||
} else {
|
||||
glog.Errorf("MovePath %s to %s: sourceInode %d not found", sourcePath, targetPath, sourceInode)
|
||||
}
|
||||
|
||||
@@ -23,23 +23,25 @@ type MetaCache struct {
|
||||
localStore filer.VirtualFilerStore
|
||||
leveldbStore *leveldb.LevelDBStore // direct reference for batch operations
|
||||
sync.RWMutex
|
||||
uidGidMapper *UidGidMapper
|
||||
markCachedFn func(fullpath util.FullPath)
|
||||
isCachedFn func(fullpath util.FullPath) bool
|
||||
invalidateFunc func(fullpath util.FullPath, entry *filer_pb.Entry)
|
||||
visitGroup singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path
|
||||
uidGidMapper *UidGidMapper
|
||||
markCachedFn func(fullpath util.FullPath)
|
||||
isCachedFn func(fullpath util.FullPath) bool
|
||||
invalidateFunc func(fullpath util.FullPath, entry *filer_pb.Entry)
|
||||
onDirectoryUpdate func(dir util.FullPath)
|
||||
visitGroup singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path
|
||||
}
|
||||
|
||||
func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath,
|
||||
markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry)) *MetaCache {
|
||||
markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry), onDirectoryUpdate func(dir util.FullPath)) *MetaCache {
|
||||
leveldbStore, virtualStore := openMetaStore(dbFolder)
|
||||
return &MetaCache{
|
||||
root: root,
|
||||
localStore: virtualStore,
|
||||
leveldbStore: leveldbStore,
|
||||
markCachedFn: markCachedFn,
|
||||
isCachedFn: isCachedFn,
|
||||
uidGidMapper: uidGidMapper,
|
||||
root: root,
|
||||
localStore: virtualStore,
|
||||
leveldbStore: leveldbStore,
|
||||
markCachedFn: markCachedFn,
|
||||
isCachedFn: isCachedFn,
|
||||
uidGidMapper: uidGidMapper,
|
||||
onDirectoryUpdate: onDirectoryUpdate,
|
||||
invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) {
|
||||
invalidateFunc(fullpath, entry)
|
||||
},
|
||||
@@ -193,3 +195,9 @@ func (mc *MetaCache) Debug() {
|
||||
func (mc *MetaCache) IsDirectoryCached(dirPath util.FullPath) bool {
|
||||
return mc.isCachedFn(dirPath)
|
||||
}
|
||||
|
||||
func (mc *MetaCache) noteDirectoryUpdate(dirPath util.FullPath) {
|
||||
if mc.onDirectoryUpdate != nil {
|
||||
mc.onDirectoryUpdate(dirPath)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ func mergeProcessors(mainProcessor func(resp *filer_pb.SubscribeMetadataResponse
|
||||
}
|
||||
}
|
||||
|
||||
func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64, followers ...*MetadataFollower) error {
|
||||
func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64, onRetry func(lastTsNs int64, err error), followers ...*MetadataFollower) error {
|
||||
|
||||
var prefixes []string
|
||||
for _, follower := range followers {
|
||||
@@ -77,6 +77,24 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
|
||||
}
|
||||
err := mc.AtomicUpdateEntryFromFiler(context.Background(), oldPath, newEntry)
|
||||
if err == nil {
|
||||
if message.NewEntry != nil || message.OldEntry != nil {
|
||||
dirsToNotify := make(map[util.FullPath]struct{})
|
||||
if oldPath != "" {
|
||||
parent, _ := oldPath.DirAndName()
|
||||
dirsToNotify[util.FullPath(parent)] = struct{}{}
|
||||
}
|
||||
if newEntry != nil {
|
||||
newParent, _ := newEntry.DirAndName()
|
||||
dirsToNotify[util.FullPath(newParent)] = struct{}{}
|
||||
}
|
||||
if message.NewEntry != nil && message.NewEntry.IsDirectory {
|
||||
childPath := util.NewFullPath(dir, message.NewEntry.Name)
|
||||
dirsToNotify[childPath] = struct{}{}
|
||||
}
|
||||
for dirPath := range dirsToNotify {
|
||||
mc.noteDirectoryUpdate(dirPath)
|
||||
}
|
||||
}
|
||||
if message.OldEntry != nil && message.NewEntry != nil {
|
||||
oldKey := util.NewFullPath(resp.Directory, message.OldEntry.Name)
|
||||
mc.invalidateFunc(oldKey, message.OldEntry)
|
||||
@@ -117,6 +135,9 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
|
||||
metadataFollowOption.ClientEpoch++
|
||||
return pb.WithFilerClientFollowMetadata(client, metadataFollowOption, mergeProcessors(processEventFn, followers...))
|
||||
}, func(err error) bool {
|
||||
if onRetry != nil {
|
||||
onRetry(metadataFollowOption.StartTsNs, err)
|
||||
}
|
||||
glog.Errorf("follow metadata updates: %v", err)
|
||||
return true
|
||||
})
|
||||
|
||||
+93
-9
@@ -75,6 +75,9 @@ type Option struct {
|
||||
RdmaMaxConcurrent int
|
||||
RdmaTimeoutMs int
|
||||
|
||||
// Directory cache refresh/eviction controls
|
||||
DirIdleEvictSec int
|
||||
|
||||
uniqueCacheDirForRead string
|
||||
uniqueCacheDirForWrite string
|
||||
}
|
||||
@@ -102,8 +105,19 @@ type WFS struct {
|
||||
rdmaClient *RDMAMountClient
|
||||
FilerConf *filer.FilerConf
|
||||
filerClient *wdclient.FilerClient // Cached volume location client
|
||||
refreshMu sync.Mutex
|
||||
refreshingDirs map[util.FullPath]struct{}
|
||||
dirHotWindow time.Duration
|
||||
dirHotThreshold int
|
||||
dirIdleEvict time.Duration
|
||||
}
|
||||
|
||||
const (
|
||||
defaultDirHotWindow = 2 * time.Second
|
||||
defaultDirHotThreshold = 64
|
||||
defaultDirIdleEvict = 10 * time.Minute
|
||||
)
|
||||
|
||||
func NewSeaweedFileSystem(option *Option) *WFS {
|
||||
// Only create FilerClient for direct volume access modes
|
||||
// When VolumeServerAccess == "filerProxy", all reads go through filer, so no volume lookup needed
|
||||
@@ -127,15 +141,28 @@ func NewSeaweedFileSystem(option *Option) *WFS {
|
||||
)
|
||||
}
|
||||
|
||||
dirHotWindow := defaultDirHotWindow
|
||||
dirHotThreshold := defaultDirHotThreshold
|
||||
dirIdleEvict := defaultDirIdleEvict
|
||||
if option.DirIdleEvictSec != 0 {
|
||||
dirIdleEvict = time.Duration(option.DirIdleEvictSec) * time.Second
|
||||
} else {
|
||||
dirIdleEvict = 0
|
||||
}
|
||||
|
||||
wfs := &WFS{
|
||||
RawFileSystem: fuse.NewDefaultRawFileSystem(),
|
||||
option: option,
|
||||
signature: util.RandomInt32(),
|
||||
inodeToPath: NewInodeToPath(util.FullPath(option.FilerMountRootPath), option.CacheMetaTTlSec),
|
||||
fhMap: NewFileHandleToInode(),
|
||||
dhMap: NewDirectoryHandleToInode(),
|
||||
filerClient: filerClient, // nil for proxy mode, initialized for direct access
|
||||
fhLockTable: util.NewLockTable[FileHandleId](),
|
||||
RawFileSystem: fuse.NewDefaultRawFileSystem(),
|
||||
option: option,
|
||||
signature: util.RandomInt32(),
|
||||
inodeToPath: NewInodeToPath(util.FullPath(option.FilerMountRootPath), option.CacheMetaTTlSec),
|
||||
fhMap: NewFileHandleToInode(),
|
||||
dhMap: NewDirectoryHandleToInode(),
|
||||
filerClient: filerClient, // nil for proxy mode, initialized for direct access
|
||||
fhLockTable: util.NewLockTable[FileHandleId](),
|
||||
refreshingDirs: make(map[util.FullPath]struct{}),
|
||||
dirHotWindow: dirHotWindow,
|
||||
dirHotThreshold: dirHotThreshold,
|
||||
dirIdleEvict: dirIdleEvict,
|
||||
}
|
||||
|
||||
wfs.option.filerIndex = int32(rand.IntN(len(option.FilerAddresses)))
|
||||
@@ -171,6 +198,10 @@ func NewSeaweedFileSystem(option *Option) *WFS {
|
||||
}
|
||||
}
|
||||
}
|
||||
}, func(dirPath util.FullPath) {
|
||||
if wfs.inodeToPath.RecordDirectoryUpdate(dirPath, time.Now(), wfs.dirHotWindow, wfs.dirHotThreshold) {
|
||||
wfs.maybeRefreshDirectory(dirPath)
|
||||
}
|
||||
})
|
||||
grace.OnInterrupt(func() {
|
||||
wfs.metaCache.Shutdown()
|
||||
@@ -215,9 +246,16 @@ func (wfs *WFS) StartBackgroundTasks() error {
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano(), follower)
|
||||
go meta_cache.SubscribeMetaEvents(wfs.metaCache, wfs.signature, wfs, wfs.option.FilerMountRootPath, startTime.UnixNano(), func(lastTsNs int64, err error) {
|
||||
glog.Warningf("meta events follow retry from %v: %v", time.Unix(0, lastTsNs), err)
|
||||
if deleteErr := wfs.metaCache.DeleteFolderChildren(context.Background(), util.FullPath(wfs.option.FilerMountRootPath)); deleteErr != nil {
|
||||
glog.Warningf("meta cache cleanup failed: %v", deleteErr)
|
||||
}
|
||||
wfs.inodeToPath.InvalidateAllChildrenCache()
|
||||
}, follower)
|
||||
go wfs.loopCheckQuota()
|
||||
go wfs.loopFlushDirtyMetadata()
|
||||
go wfs.loopEvictIdleDirCache()
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -306,6 +344,9 @@ func (wfs *WFS) lookupEntry(fullpath util.FullPath) (*filer.Entry, fuse.Status)
|
||||
glog.V(1).Infof("lookupEntry GetEntry %s: %v", fullpath, err)
|
||||
return nil, fuse.ENOENT
|
||||
}
|
||||
if entry != nil && entry.Attributes != nil && wfs.option.UidGidMapper != nil {
|
||||
entry.Attributes.Uid, entry.Attributes.Gid = wfs.option.UidGidMapper.FilerToLocal(entry.Attributes.Uid, entry.Attributes.Gid)
|
||||
}
|
||||
return filer.FromPbEntry(dir, entry), fuse.OK
|
||||
}
|
||||
|
||||
@@ -330,6 +371,49 @@ func (wfs *WFS) ClearCacheDir() {
|
||||
os.RemoveAll(wfs.option.getUniqueCacheDirForRead())
|
||||
}
|
||||
|
||||
func (wfs *WFS) maybeRefreshDirectory(dirPath util.FullPath) {
|
||||
if !wfs.inodeToPath.NeedsRefresh(dirPath) {
|
||||
return
|
||||
}
|
||||
wfs.refreshMu.Lock()
|
||||
if _, exists := wfs.refreshingDirs[dirPath]; exists {
|
||||
wfs.refreshMu.Unlock()
|
||||
return
|
||||
}
|
||||
wfs.refreshingDirs[dirPath] = struct{}{}
|
||||
wfs.refreshMu.Unlock()
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
wfs.refreshMu.Lock()
|
||||
delete(wfs.refreshingDirs, dirPath)
|
||||
wfs.refreshMu.Unlock()
|
||||
}()
|
||||
wfs.inodeToPath.InvalidateChildrenCache(dirPath)
|
||||
if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil {
|
||||
glog.Warningf("refresh dir cache %s: %v", dirPath, err)
|
||||
return
|
||||
}
|
||||
wfs.inodeToPath.MarkDirectoryRefreshed(dirPath, time.Now())
|
||||
}()
|
||||
}
|
||||
|
||||
func (wfs *WFS) loopEvictIdleDirCache() {
|
||||
if wfs.dirIdleEvict <= 0 {
|
||||
return
|
||||
}
|
||||
ticker := time.NewTicker(wfs.dirIdleEvict / 2)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
dirs := wfs.inodeToPath.CollectEvictableDirs(time.Now(), wfs.dirIdleEvict)
|
||||
for _, dir := range dirs {
|
||||
if err := wfs.metaCache.DeleteFolderChildren(context.Background(), dir); err != nil {
|
||||
glog.V(2).Infof("evict dir cache %s: %v", dir, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (option *Option) setupUniqueCacheDirectory() {
|
||||
cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + version.Version()))[0:8]
|
||||
option.uniqueCacheDirForRead = path.Join(option.CacheDirForRead, cacheUniqueId)
|
||||
|
||||
@@ -71,6 +71,7 @@ func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out
|
||||
// Only cache the entry if the parent directory is already cached.
|
||||
// This avoids polluting the cache with partial directory data.
|
||||
if wfs.metaCache.IsDirectoryCached(dirFullPath) {
|
||||
wfs.inodeToPath.TouchDirectory(dirFullPath)
|
||||
if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
|
||||
return fmt.Errorf("local mkdir dir %s: %w", entryFullPath, err)
|
||||
}
|
||||
@@ -122,6 +123,7 @@ func (wfs *WFS) Rmdir(cancel <-chan struct{}, header *fuse.InHeader, name string
|
||||
|
||||
wfs.metaCache.DeleteEntry(context.Background(), entryFullPath)
|
||||
wfs.inodeToPath.RemovePath(entryFullPath)
|
||||
wfs.inodeToPath.TouchDirectory(dirFullPath)
|
||||
|
||||
return fuse.OK
|
||||
|
||||
|
||||
@@ -163,6 +163,8 @@ func (wfs *WFS) doReadDirectory(input *fuse.ReadIn, out *fuse.DirEntryList, isPl
|
||||
if code != fuse.OK {
|
||||
return code
|
||||
}
|
||||
wfs.inodeToPath.TouchDirectory(dirPath)
|
||||
wfs.maybeRefreshDirectory(dirPath)
|
||||
|
||||
var dirEntry fuse.DirEntry
|
||||
|
||||
|
||||
@@ -91,6 +91,7 @@ func (wfs *WFS) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, name string, out
|
||||
// Only cache the entry if the parent directory is already cached.
|
||||
// This avoids polluting the cache with partial directory data.
|
||||
if wfs.metaCache.IsDirectoryCached(dirFullPath) {
|
||||
wfs.inodeToPath.TouchDirectory(dirFullPath)
|
||||
if err := wfs.metaCache.InsertEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry)); err != nil {
|
||||
return fmt.Errorf("local mknod %s: %w", entryFullPath, err)
|
||||
}
|
||||
@@ -153,6 +154,7 @@ func (wfs *WFS) Unlink(cancel <-chan struct{}, header *fuse.InHeader, name strin
|
||||
glog.V(3).Infof("local DeleteEntry %s: %v", entryFullPath, err)
|
||||
return fuse.EIO
|
||||
}
|
||||
wfs.inodeToPath.TouchDirectory(dirFullPath)
|
||||
|
||||
wfs.inodeToPath.RemovePath(entryFullPath)
|
||||
|
||||
|
||||
@@ -220,6 +220,8 @@ func (wfs *WFS) Rename(cancel <-chan struct{}, in *fuse.RenameIn, oldName string
|
||||
glog.V(0).Infof("Link: %v", err)
|
||||
return
|
||||
}
|
||||
wfs.inodeToPath.TouchDirectory(oldDir)
|
||||
wfs.inodeToPath.TouchDirectory(newDir)
|
||||
|
||||
return fuse.OK
|
||||
|
||||
|
||||
@@ -292,7 +292,7 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option
|
||||
uploadResult.Name = option.Filename
|
||||
uploadResult.Mime = option.MimeType
|
||||
uploadResult.CipherKey = cipherKey
|
||||
uploadResult.Size = uint32(len(data))
|
||||
uploadResult.Size = uint32(clearDataLen)
|
||||
if contentIsGzipped {
|
||||
uploadResult.Gzip = 1
|
||||
}
|
||||
|
||||
@@ -53,7 +53,11 @@ service VolumeServer {
|
||||
}
|
||||
rpc VolumeStatus (VolumeStatusRequest) returns (VolumeStatusResponse) {
|
||||
}
|
||||
// TODO(issues/7977): add RPCs to control state flags
|
||||
|
||||
rpc GetState (GetStateRequest) returns (GetStateResponse) {
|
||||
}
|
||||
rpc SetState (SetStateRequest) returns (SetStateResponse) {
|
||||
}
|
||||
|
||||
// copy the .idx .dat files, and mount this volume
|
||||
rpc VolumeCopy (VolumeCopyRequest) returns (stream VolumeCopyResponse) {
|
||||
@@ -116,6 +120,12 @@ service VolumeServer {
|
||||
rpc FetchAndWriteNeedle (FetchAndWriteNeedleRequest) returns (FetchAndWriteNeedleResponse) {
|
||||
}
|
||||
|
||||
// scrubbing
|
||||
rpc ScrubVolume (ScrubVolumeRequest) returns (ScrubVolumeResponse) {
|
||||
}
|
||||
rpc ScrubEcVolume (ScrubEcVolumeRequest) returns (ScrubEcVolumeResponse) {
|
||||
}
|
||||
|
||||
// <experimental> query
|
||||
rpc Query (QueryRequest) returns (stream QueriedStripe) {
|
||||
}
|
||||
@@ -270,6 +280,21 @@ message VolumeStatusResponse {
|
||||
uint64 file_deleted_count = 4;
|
||||
}
|
||||
|
||||
message GetStateRequest {
|
||||
}
|
||||
message GetStateResponse {
|
||||
VolumeServerState state = 1;
|
||||
}
|
||||
|
||||
message SetStateRequest {
|
||||
// SetState updates *all* volume server flags at once. Retrieve state with GetState(),
|
||||
// modify individual flags as required, then call this RPC to update.
|
||||
VolumeServerState state = 1;
|
||||
}
|
||||
message SetStateResponse {
|
||||
VolumeServerState state = 1;
|
||||
}
|
||||
|
||||
message VolumeCopyRequest {
|
||||
uint32 volume_id = 1;
|
||||
string collection = 2;
|
||||
@@ -477,6 +502,7 @@ message EcShardInfo {
|
||||
uint32 shard_id = 1;
|
||||
int64 size = 2;
|
||||
string collection = 3;
|
||||
uint32 volume_id = 4;
|
||||
}
|
||||
|
||||
message ReadVolumeFileStatusRequest {
|
||||
@@ -613,6 +639,37 @@ message FetchAndWriteNeedleResponse {
|
||||
string e_tag = 1;
|
||||
}
|
||||
|
||||
enum VolumeScrubMode {
|
||||
UNKNOWN = 0;
|
||||
INDEX = 1;
|
||||
FULL = 2;
|
||||
}
|
||||
|
||||
message ScrubVolumeRequest {
|
||||
VolumeScrubMode mode = 1;
|
||||
// optional list of volume IDs to scrub. if empty, all volumes for the server are scrubbed.
|
||||
repeated uint32 volume_ids = 2;
|
||||
}
|
||||
message ScrubVolumeResponse {
|
||||
uint64 total_volumes = 1;
|
||||
uint64 total_files = 2;
|
||||
repeated uint32 broken_volume_ids = 3;
|
||||
repeated string details = 4;
|
||||
}
|
||||
|
||||
message ScrubEcVolumeRequest {
|
||||
VolumeScrubMode mode = 1;
|
||||
// optional list of volume IDs to scrub. if empty, all EC volumes for the server are scrubbed.
|
||||
repeated uint32 volume_ids = 2;
|
||||
}
|
||||
message ScrubEcVolumeResponse {
|
||||
uint64 total_volumes = 1;
|
||||
uint64 total_files = 2;
|
||||
repeated uint32 broken_volume_ids = 3;
|
||||
repeated EcShardInfo broken_shard_infos = 4;
|
||||
repeated string details = 5;
|
||||
}
|
||||
|
||||
// select on volume servers
|
||||
message QueryRequest {
|
||||
repeated string selections = 1;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,7 +1,7 @@
|
||||
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
|
||||
// versions:
|
||||
// - protoc-gen-go-grpc v1.5.1
|
||||
// - protoc v6.33.4
|
||||
// - protoc-gen-go-grpc v1.6.0
|
||||
// - protoc v3.21.12
|
||||
// source: volume_server.proto
|
||||
|
||||
package volume_server_pb
|
||||
@@ -35,6 +35,8 @@ const (
|
||||
VolumeServer_VolumeMarkWritable_FullMethodName = "/volume_server_pb.VolumeServer/VolumeMarkWritable"
|
||||
VolumeServer_VolumeConfigure_FullMethodName = "/volume_server_pb.VolumeServer/VolumeConfigure"
|
||||
VolumeServer_VolumeStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeStatus"
|
||||
VolumeServer_GetState_FullMethodName = "/volume_server_pb.VolumeServer/GetState"
|
||||
VolumeServer_SetState_FullMethodName = "/volume_server_pb.VolumeServer/SetState"
|
||||
VolumeServer_VolumeCopy_FullMethodName = "/volume_server_pb.VolumeServer/VolumeCopy"
|
||||
VolumeServer_ReadVolumeFileStatus_FullMethodName = "/volume_server_pb.VolumeServer/ReadVolumeFileStatus"
|
||||
VolumeServer_CopyFile_FullMethodName = "/volume_server_pb.VolumeServer/CopyFile"
|
||||
@@ -60,6 +62,8 @@ const (
|
||||
VolumeServer_VolumeServerStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerStatus"
|
||||
VolumeServer_VolumeServerLeave_FullMethodName = "/volume_server_pb.VolumeServer/VolumeServerLeave"
|
||||
VolumeServer_FetchAndWriteNeedle_FullMethodName = "/volume_server_pb.VolumeServer/FetchAndWriteNeedle"
|
||||
VolumeServer_ScrubVolume_FullMethodName = "/volume_server_pb.VolumeServer/ScrubVolume"
|
||||
VolumeServer_ScrubEcVolume_FullMethodName = "/volume_server_pb.VolumeServer/ScrubEcVolume"
|
||||
VolumeServer_Query_FullMethodName = "/volume_server_pb.VolumeServer/Query"
|
||||
VolumeServer_VolumeNeedleStatus_FullMethodName = "/volume_server_pb.VolumeServer/VolumeNeedleStatus"
|
||||
VolumeServer_Ping_FullMethodName = "/volume_server_pb.VolumeServer/Ping"
|
||||
@@ -86,6 +90,8 @@ type VolumeServerClient interface {
|
||||
VolumeMarkWritable(ctx context.Context, in *VolumeMarkWritableRequest, opts ...grpc.CallOption) (*VolumeMarkWritableResponse, error)
|
||||
VolumeConfigure(ctx context.Context, in *VolumeConfigureRequest, opts ...grpc.CallOption) (*VolumeConfigureResponse, error)
|
||||
VolumeStatus(ctx context.Context, in *VolumeStatusRequest, opts ...grpc.CallOption) (*VolumeStatusResponse, error)
|
||||
GetState(ctx context.Context, in *GetStateRequest, opts ...grpc.CallOption) (*GetStateResponse, error)
|
||||
SetState(ctx context.Context, in *SetStateRequest, opts ...grpc.CallOption) (*SetStateResponse, error)
|
||||
// copy the .idx .dat files, and mount this volume
|
||||
VolumeCopy(ctx context.Context, in *VolumeCopyRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[VolumeCopyResponse], error)
|
||||
ReadVolumeFileStatus(ctx context.Context, in *ReadVolumeFileStatusRequest, opts ...grpc.CallOption) (*ReadVolumeFileStatusResponse, error)
|
||||
@@ -115,6 +121,9 @@ type VolumeServerClient interface {
|
||||
VolumeServerLeave(ctx context.Context, in *VolumeServerLeaveRequest, opts ...grpc.CallOption) (*VolumeServerLeaveResponse, error)
|
||||
// remote storage
|
||||
FetchAndWriteNeedle(ctx context.Context, in *FetchAndWriteNeedleRequest, opts ...grpc.CallOption) (*FetchAndWriteNeedleResponse, error)
|
||||
// scrubbing
|
||||
ScrubVolume(ctx context.Context, in *ScrubVolumeRequest, opts ...grpc.CallOption) (*ScrubVolumeResponse, error)
|
||||
ScrubEcVolume(ctx context.Context, in *ScrubEcVolumeRequest, opts ...grpc.CallOption) (*ScrubEcVolumeResponse, error)
|
||||
// <experimental> query
|
||||
Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueriedStripe], error)
|
||||
VolumeNeedleStatus(ctx context.Context, in *VolumeNeedleStatusRequest, opts ...grpc.CallOption) (*VolumeNeedleStatusResponse, error)
|
||||
@@ -307,6 +316,26 @@ func (c *volumeServerClient) VolumeStatus(ctx context.Context, in *VolumeStatusR
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeServerClient) GetState(ctx context.Context, in *GetStateRequest, opts ...grpc.CallOption) (*GetStateResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(GetStateResponse)
|
||||
err := c.cc.Invoke(ctx, VolumeServer_GetState_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeServerClient) SetState(ctx context.Context, in *SetStateRequest, opts ...grpc.CallOption) (*SetStateResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(SetStateResponse)
|
||||
err := c.cc.Invoke(ctx, VolumeServer_SetState_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeServerClient) VolumeCopy(ctx context.Context, in *VolumeCopyRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[VolumeCopyResponse], error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[2], VolumeServer_VolumeCopy_FullMethodName, cOpts...)
|
||||
@@ -623,6 +652,26 @@ func (c *volumeServerClient) FetchAndWriteNeedle(ctx context.Context, in *FetchA
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeServerClient) ScrubVolume(ctx context.Context, in *ScrubVolumeRequest, opts ...grpc.CallOption) (*ScrubVolumeResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ScrubVolumeResponse)
|
||||
err := c.cc.Invoke(ctx, VolumeServer_ScrubVolume_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeServerClient) ScrubEcVolume(ctx context.Context, in *ScrubEcVolumeRequest, opts ...grpc.CallOption) (*ScrubEcVolumeResponse, error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
out := new(ScrubEcVolumeResponse)
|
||||
err := c.cc.Invoke(ctx, VolumeServer_ScrubEcVolume_FullMethodName, in, out, cOpts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *volumeServerClient) Query(ctx context.Context, in *QueryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueriedStripe], error) {
|
||||
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
|
||||
stream, err := c.cc.NewStream(ctx, &VolumeServer_ServiceDesc.Streams[10], VolumeServer_Query_FullMethodName, cOpts...)
|
||||
@@ -683,6 +732,8 @@ type VolumeServerServer interface {
|
||||
VolumeMarkWritable(context.Context, *VolumeMarkWritableRequest) (*VolumeMarkWritableResponse, error)
|
||||
VolumeConfigure(context.Context, *VolumeConfigureRequest) (*VolumeConfigureResponse, error)
|
||||
VolumeStatus(context.Context, *VolumeStatusRequest) (*VolumeStatusResponse, error)
|
||||
GetState(context.Context, *GetStateRequest) (*GetStateResponse, error)
|
||||
SetState(context.Context, *SetStateRequest) (*SetStateResponse, error)
|
||||
// copy the .idx .dat files, and mount this volume
|
||||
VolumeCopy(*VolumeCopyRequest, grpc.ServerStreamingServer[VolumeCopyResponse]) error
|
||||
ReadVolumeFileStatus(context.Context, *ReadVolumeFileStatusRequest) (*ReadVolumeFileStatusResponse, error)
|
||||
@@ -712,6 +763,9 @@ type VolumeServerServer interface {
|
||||
VolumeServerLeave(context.Context, *VolumeServerLeaveRequest) (*VolumeServerLeaveResponse, error)
|
||||
// remote storage
|
||||
FetchAndWriteNeedle(context.Context, *FetchAndWriteNeedleRequest) (*FetchAndWriteNeedleResponse, error)
|
||||
// scrubbing
|
||||
ScrubVolume(context.Context, *ScrubVolumeRequest) (*ScrubVolumeResponse, error)
|
||||
ScrubEcVolume(context.Context, *ScrubEcVolumeRequest) (*ScrubEcVolumeResponse, error)
|
||||
// <experimental> query
|
||||
Query(*QueryRequest, grpc.ServerStreamingServer[QueriedStripe]) error
|
||||
VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error)
|
||||
@@ -727,136 +781,148 @@ type VolumeServerServer interface {
|
||||
type UnimplementedVolumeServerServer struct{}
|
||||
|
||||
func (UnimplementedVolumeServerServer) BatchDelete(context.Context, *BatchDeleteRequest) (*BatchDeleteResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method BatchDelete not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method BatchDelete not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VacuumVolumeCheck(context.Context, *VacuumVolumeCheckRequest) (*VacuumVolumeCheckResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCheck not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VacuumVolumeCheck not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VacuumVolumeCompact(*VacuumVolumeCompactRequest, grpc.ServerStreamingServer[VacuumVolumeCompactResponse]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method VacuumVolumeCompact not implemented")
|
||||
return status.Error(codes.Unimplemented, "method VacuumVolumeCompact not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VacuumVolumeCommit(context.Context, *VacuumVolumeCommitRequest) (*VacuumVolumeCommitResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCommit not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VacuumVolumeCommit not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VacuumVolumeCleanup(context.Context, *VacuumVolumeCleanupRequest) (*VacuumVolumeCleanupResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VacuumVolumeCleanup not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VacuumVolumeCleanup not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method DeleteCollection not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method DeleteCollection not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) AllocateVolume(context.Context, *AllocateVolumeRequest) (*AllocateVolumeResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method AllocateVolume not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method AllocateVolume not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeSyncStatus not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeSyncStatus not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeIncrementalCopy(*VolumeIncrementalCopyRequest, grpc.ServerStreamingServer[VolumeIncrementalCopyResponse]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method VolumeIncrementalCopy not implemented")
|
||||
return status.Error(codes.Unimplemented, "method VolumeIncrementalCopy not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeMount not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeMount not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeUnmount(context.Context, *VolumeUnmountRequest) (*VolumeUnmountResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeUnmount not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeUnmount not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeDelete(context.Context, *VolumeDeleteRequest) (*VolumeDeleteResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeDelete not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeDelete not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeMarkReadonly(context.Context, *VolumeMarkReadonlyRequest) (*VolumeMarkReadonlyResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeMarkReadonly not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeMarkReadonly not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeMarkWritable(context.Context, *VolumeMarkWritableRequest) (*VolumeMarkWritableResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeMarkWritable not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeMarkWritable not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeConfigure(context.Context, *VolumeConfigureRequest) (*VolumeConfigureResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeConfigure not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeConfigure not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeStatus(context.Context, *VolumeStatusRequest) (*VolumeStatusResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeStatus not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeStatus not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) GetState(context.Context, *GetStateRequest) (*GetStateResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method GetState not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) SetState(context.Context, *SetStateRequest) (*SetStateResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method SetState not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeCopy(*VolumeCopyRequest, grpc.ServerStreamingServer[VolumeCopyResponse]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method VolumeCopy not implemented")
|
||||
return status.Error(codes.Unimplemented, "method VolumeCopy not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) ReadVolumeFileStatus(context.Context, *ReadVolumeFileStatusRequest) (*ReadVolumeFileStatusResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ReadVolumeFileStatus not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method ReadVolumeFileStatus not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) CopyFile(*CopyFileRequest, grpc.ServerStreamingServer[CopyFileResponse]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method CopyFile not implemented")
|
||||
return status.Error(codes.Unimplemented, "method CopyFile not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) ReceiveFile(grpc.ClientStreamingServer[ReceiveFileRequest, ReceiveFileResponse]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method ReceiveFile not implemented")
|
||||
return status.Error(codes.Unimplemented, "method ReceiveFile not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) ReadNeedleBlob(context.Context, *ReadNeedleBlobRequest) (*ReadNeedleBlobResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ReadNeedleBlob not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method ReadNeedleBlob not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) ReadNeedleMeta(context.Context, *ReadNeedleMetaRequest) (*ReadNeedleMetaResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ReadNeedleMeta not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method ReadNeedleMeta not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) WriteNeedleBlob(context.Context, *WriteNeedleBlobRequest) (*WriteNeedleBlobResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method WriteNeedleBlob not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method WriteNeedleBlob not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) ReadAllNeedles(*ReadAllNeedlesRequest, grpc.ServerStreamingServer[ReadAllNeedlesResponse]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method ReadAllNeedles not implemented")
|
||||
return status.Error(codes.Unimplemented, "method ReadAllNeedles not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeTailSender(*VolumeTailSenderRequest, grpc.ServerStreamingServer[VolumeTailSenderResponse]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method VolumeTailSender not implemented")
|
||||
return status.Error(codes.Unimplemented, "method VolumeTailSender not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeTailReceiver(context.Context, *VolumeTailReceiverRequest) (*VolumeTailReceiverResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeTailReceiver not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeTailReceiver not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeEcShardsGenerate(context.Context, *VolumeEcShardsGenerateRequest) (*VolumeEcShardsGenerateResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsGenerate not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsGenerate not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeEcShardsRebuild(context.Context, *VolumeEcShardsRebuildRequest) (*VolumeEcShardsRebuildResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsRebuild not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsRebuild not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeEcShardsCopy(context.Context, *VolumeEcShardsCopyRequest) (*VolumeEcShardsCopyResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsCopy not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsCopy not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeEcShardsDelete(context.Context, *VolumeEcShardsDeleteRequest) (*VolumeEcShardsDeleteResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsDelete not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsDelete not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeEcShardsMount(context.Context, *VolumeEcShardsMountRequest) (*VolumeEcShardsMountResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsMount not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsMount not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeEcShardsUnmount(context.Context, *VolumeEcShardsUnmountRequest) (*VolumeEcShardsUnmountResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsUnmount not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsUnmount not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeEcShardRead(*VolumeEcShardReadRequest, grpc.ServerStreamingServer[VolumeEcShardReadResponse]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method VolumeEcShardRead not implemented")
|
||||
return status.Error(codes.Unimplemented, "method VolumeEcShardRead not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeEcBlobDelete(context.Context, *VolumeEcBlobDeleteRequest) (*VolumeEcBlobDeleteResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeEcBlobDelete not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeEcBlobDelete not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeEcShardsToVolume(context.Context, *VolumeEcShardsToVolumeRequest) (*VolumeEcShardsToVolumeResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsToVolume not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsToVolume not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeEcShardsInfo(context.Context, *VolumeEcShardsInfoRequest) (*VolumeEcShardsInfoResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeEcShardsInfo not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeEcShardsInfo not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeTierMoveDatToRemote(*VolumeTierMoveDatToRemoteRequest, grpc.ServerStreamingServer[VolumeTierMoveDatToRemoteResponse]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method VolumeTierMoveDatToRemote not implemented")
|
||||
return status.Error(codes.Unimplemented, "method VolumeTierMoveDatToRemote not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeTierMoveDatFromRemote(*VolumeTierMoveDatFromRemoteRequest, grpc.ServerStreamingServer[VolumeTierMoveDatFromRemoteResponse]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method VolumeTierMoveDatFromRemote not implemented")
|
||||
return status.Error(codes.Unimplemented, "method VolumeTierMoveDatFromRemote not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeServerStatus(context.Context, *VolumeServerStatusRequest) (*VolumeServerStatusResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeServerStatus not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeServerStatus not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeServerLeave(context.Context, *VolumeServerLeaveRequest) (*VolumeServerLeaveResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeServerLeave not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeServerLeave not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) FetchAndWriteNeedle(context.Context, *FetchAndWriteNeedleRequest) (*FetchAndWriteNeedleResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method FetchAndWriteNeedle not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method FetchAndWriteNeedle not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) ScrubVolume(context.Context, *ScrubVolumeRequest) (*ScrubVolumeResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method ScrubVolume not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) ScrubEcVolume(context.Context, *ScrubEcVolumeRequest) (*ScrubEcVolumeResponse, error) {
|
||||
return nil, status.Error(codes.Unimplemented, "method ScrubEcVolume not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) Query(*QueryRequest, grpc.ServerStreamingServer[QueriedStripe]) error {
|
||||
return status.Errorf(codes.Unimplemented, "method Query not implemented")
|
||||
return status.Error(codes.Unimplemented, "method Query not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) VolumeNeedleStatus(context.Context, *VolumeNeedleStatusRequest) (*VolumeNeedleStatusResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method VolumeNeedleStatus not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method VolumeNeedleStatus not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) Ping(context.Context, *PingRequest) (*PingResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
|
||||
return nil, status.Error(codes.Unimplemented, "method Ping not implemented")
|
||||
}
|
||||
func (UnimplementedVolumeServerServer) mustEmbedUnimplementedVolumeServerServer() {}
|
||||
func (UnimplementedVolumeServerServer) testEmbeddedByValue() {}
|
||||
@@ -869,7 +935,7 @@ type UnsafeVolumeServerServer interface {
|
||||
}
|
||||
|
||||
func RegisterVolumeServerServer(s grpc.ServiceRegistrar, srv VolumeServerServer) {
|
||||
// If the following call pancis, it indicates UnimplementedVolumeServerServer was
|
||||
// If the following call panics, it indicates UnimplementedVolumeServerServer was
|
||||
// embedded by pointer and is nil. This will cause panics if an
|
||||
// unimplemented method is ever invoked, so we test this at initialization
|
||||
// time to prevent it from happening at runtime later due to I/O.
|
||||
@@ -1153,6 +1219,42 @@ func _VolumeServer_VolumeStatus_Handler(srv interface{}, ctx context.Context, de
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _VolumeServer_GetState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(GetStateRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VolumeServerServer).GetState(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: VolumeServer_GetState_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VolumeServerServer).GetState(ctx, req.(*GetStateRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _VolumeServer_SetState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(SetStateRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VolumeServerServer).SetState(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: VolumeServer_SetState_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VolumeServerServer).SetState(ctx, req.(*SetStateRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _VolumeServer_VolumeCopy_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||
m := new(VolumeCopyRequest)
|
||||
if err := stream.RecvMsg(m); err != nil {
|
||||
@@ -1543,6 +1645,42 @@ func _VolumeServer_FetchAndWriteNeedle_Handler(srv interface{}, ctx context.Cont
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _VolumeServer_ScrubVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ScrubVolumeRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VolumeServerServer).ScrubVolume(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: VolumeServer_ScrubVolume_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VolumeServerServer).ScrubVolume(ctx, req.(*ScrubVolumeRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _VolumeServer_ScrubEcVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ScrubEcVolumeRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(VolumeServerServer).ScrubEcVolume(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: VolumeServer_ScrubEcVolume_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(VolumeServerServer).ScrubEcVolume(ctx, req.(*ScrubEcVolumeRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _VolumeServer_Query_Handler(srv interface{}, stream grpc.ServerStream) error {
|
||||
m := new(QueryRequest)
|
||||
if err := stream.RecvMsg(m); err != nil {
|
||||
@@ -1653,6 +1791,14 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{
|
||||
MethodName: "VolumeStatus",
|
||||
Handler: _VolumeServer_VolumeStatus_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "GetState",
|
||||
Handler: _VolumeServer_GetState_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "SetState",
|
||||
Handler: _VolumeServer_SetState_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ReadVolumeFileStatus",
|
||||
Handler: _VolumeServer_ReadVolumeFileStatus_Handler,
|
||||
@@ -1721,6 +1867,14 @@ var VolumeServer_ServiceDesc = grpc.ServiceDesc{
|
||||
MethodName: "FetchAndWriteNeedle",
|
||||
Handler: _VolumeServer_FetchAndWriteNeedle_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ScrubVolume",
|
||||
Handler: _VolumeServer_ScrubVolume_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ScrubEcVolume",
|
||||
Handler: _VolumeServer_ScrubEcVolume_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "VolumeNeedleStatus",
|
||||
Handler: _VolumeServer_VolumeNeedleStatus_Handler,
|
||||
|
||||
@@ -38,26 +38,26 @@ func (s gcsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.
|
||||
googleApplicationCredentials := conf.GcsGoogleApplicationCredentials
|
||||
|
||||
if googleApplicationCredentials == "" {
|
||||
found := false
|
||||
googleApplicationCredentials, found = os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS")
|
||||
if !found {
|
||||
return nil, fmt.Errorf("need to specific GOOGLE_APPLICATION_CREDENTIALS env variable")
|
||||
if creds, found := os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS"); found {
|
||||
googleApplicationCredentials = creds
|
||||
} else {
|
||||
glog.Warningf("no GOOGLE_APPLICATION_CREDENTIALS env variable found, falling back to Application Default Credentials")
|
||||
}
|
||||
}
|
||||
|
||||
projectID := conf.GcsProjectId
|
||||
if projectID == "" {
|
||||
found := false
|
||||
projectID, found = os.LookupEnv("GOOGLE_CLOUD_PROJECT")
|
||||
if !found {
|
||||
glog.Warningf("need to specific GOOGLE_CLOUD_PROJECT env variable")
|
||||
if pid, found := os.LookupEnv("GOOGLE_CLOUD_PROJECT"); found {
|
||||
projectID = pid
|
||||
} else {
|
||||
glog.Warningf("need to specify GOOGLE_CLOUD_PROJECT env variable")
|
||||
}
|
||||
}
|
||||
|
||||
googleApplicationCredentials = util.ResolvePath(googleApplicationCredentials)
|
||||
|
||||
var clientOpts []option.ClientOption
|
||||
|
||||
if googleApplicationCredentials != "" {
|
||||
googleApplicationCredentials = util.ResolvePath(googleApplicationCredentials)
|
||||
var data []byte
|
||||
var err error
|
||||
if strings.HasPrefix(googleApplicationCredentials, "{") {
|
||||
|
||||
@@ -26,6 +26,7 @@ import (
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -284,8 +285,12 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh
|
||||
}
|
||||
|
||||
// 8. Verify the signature, trying with X-Forwarded-Prefix first
|
||||
pathForSignature := r.URL.EscapedPath()
|
||||
if pathForSignature == "" {
|
||||
pathForSignature = r.URL.Path
|
||||
}
|
||||
if forwardedPrefix := r.Header.Get("X-Forwarded-Prefix"); forwardedPrefix != "" {
|
||||
cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, r.URL.Path)
|
||||
cleanedPath := buildPathWithForwardedPrefix(forwardedPrefix, pathForSignature)
|
||||
calculatedSignature, errCode = verify(cleanedPath)
|
||||
if errCode == s3err.ErrNone {
|
||||
return identity, cred, calculatedSignature, authInfo, s3err.ErrNone
|
||||
@@ -293,12 +298,20 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh
|
||||
}
|
||||
|
||||
// 9. Verify with the original path
|
||||
calculatedSignature, errCode = verify(r.URL.Path)
|
||||
if errCode != s3err.ErrNone {
|
||||
return nil, nil, "", nil, errCode
|
||||
calculatedSignature, errCode = verify(pathForSignature)
|
||||
if errCode == s3err.ErrNone {
|
||||
return identity, cred, calculatedSignature, authInfo, s3err.ErrNone
|
||||
}
|
||||
|
||||
return identity, cred, calculatedSignature, authInfo, s3err.ErrNone
|
||||
// 10. Retry with decoded path if signature used raw path encoding
|
||||
if decodedPath, decodeErr := url.PathUnescape(pathForSignature); decodeErr == nil && decodedPath != pathForSignature {
|
||||
calculatedSignature, errCode = verify(decodedPath)
|
||||
if errCode == s3err.ErrNone {
|
||||
return identity, cred, calculatedSignature, authInfo, s3err.ErrNone
|
||||
}
|
||||
}
|
||||
|
||||
return nil, nil, "", nil, errCode
|
||||
}
|
||||
|
||||
// validateSTSSessionToken validates an STS session token and extracts temporary credentials
|
||||
|
||||
@@ -1,10 +1,71 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
)
|
||||
|
||||
func TestExtractV4AuthInfoFromHeader_S3Tables(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
dateStr := now.Format(iso8601Format)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
service string
|
||||
body string
|
||||
expectAutoHash bool
|
||||
}{
|
||||
{
|
||||
name: "s3 service should not auto-hash",
|
||||
service: "s3",
|
||||
body: "hello",
|
||||
expectAutoHash: false,
|
||||
},
|
||||
{
|
||||
name: "s3tables service should auto-hash",
|
||||
service: "s3tables",
|
||||
body: "hello",
|
||||
expectAutoHash: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
body := bytes.NewReader([]byte(tt.body))
|
||||
req, _ := http.NewRequest(http.MethodPost, "http://localhost/", body)
|
||||
|
||||
authHeader := fmt.Sprintf("AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/%s/us-east-1/%s/aws4_request, SignedHeaders=host, Signature=dummy",
|
||||
now.Format(yyyymmdd), tt.service)
|
||||
req.Header.Set("Authorization", authHeader)
|
||||
req.Header.Set("x-amz-date", dateStr)
|
||||
|
||||
authInfo, errCode := extractV4AuthInfoFromHeader(req)
|
||||
if errCode != s3err.ErrNone {
|
||||
t.Fatalf("extractV4AuthInfoFromHeader failed: %v", errCode)
|
||||
}
|
||||
|
||||
if tt.expectAutoHash {
|
||||
expectedHash := sha256.Sum256([]byte(tt.body))
|
||||
expectedHashStr := hex.EncodeToString(expectedHash[:])
|
||||
if authInfo.HashedPayload != expectedHashStr {
|
||||
t.Errorf("Expected auto-hashed payload %s, got %s", expectedHashStr, authInfo.HashedPayload)
|
||||
}
|
||||
} else {
|
||||
if authInfo.HashedPayload != emptySHA256 {
|
||||
t.Errorf("Expected non-auto-hashed payload %s (emptySHA256), got %s", emptySHA256, authInfo.HashedPayload)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPathWithForwardedPrefix(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
@@ -453,7 +453,7 @@ func TestMiddlewareVaryHeader(t *testing.T) {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Setup mocks
|
||||
bucketChecker := &mockBucketChecker{bucketExists: true}
|
||||
|
||||
|
||||
var errCode s3err.ErrorCode
|
||||
if tt.bucketConfig == nil {
|
||||
errCode = s3err.ErrNoSuchCORSConfiguration
|
||||
@@ -503,7 +503,7 @@ func TestMiddlewareVaryHeader(t *testing.T) {
|
||||
func TestHandleOptionsRequestVaryHeader(t *testing.T) {
|
||||
// Setup mocks
|
||||
bucketChecker := &mockBucketChecker{bucketExists: true}
|
||||
|
||||
|
||||
config := &CORSConfiguration{
|
||||
CORSRules: []CORSRule{
|
||||
{
|
||||
@@ -528,7 +528,7 @@ func TestHandleOptionsRequestVaryHeader(t *testing.T) {
|
||||
"bucket": "testbucket",
|
||||
"object": "testobject",
|
||||
})
|
||||
|
||||
|
||||
// Set valid CORS headers
|
||||
req.Header.Set("Origin", "https://example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
|
||||
@@ -155,6 +155,20 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
|
||||
// Determine whether we can reuse the source MD5 (direct copy without encryption changes).
|
||||
canReuseSourceMd5 := false
|
||||
var sourceMd5 []byte
|
||||
if entry.Attributes != nil && len(entry.Attributes.Md5) > 0 {
|
||||
sourceMd5 = append([]byte(nil), entry.Attributes.Md5...)
|
||||
srcPath := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, srcBucket, srcObject)
|
||||
dstPath := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, dstBucket, dstObject)
|
||||
state := DetectEncryptionStateWithEntry(entry, r, srcPath, dstPath)
|
||||
s3a.applyCopyBucketDefaultEncryption(state, dstBucket)
|
||||
if strategy, err := DetermineUnifiedCopyStrategy(state, entry.Extended, r); err == nil && strategy == CopyStrategyDirect {
|
||||
canReuseSourceMd5 = true
|
||||
}
|
||||
}
|
||||
|
||||
// Create new entry for destination
|
||||
dstEntry := &filer_pb.Entry{
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
@@ -237,9 +251,17 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if dstEntry.Attributes != nil {
|
||||
if len(dstEntry.Attributes.Md5) == 0 && canReuseSourceMd5 {
|
||||
dstEntry.Attributes.Md5 = append([]byte(nil), sourceMd5...)
|
||||
} else if uint64(len(dstEntry.Content)) == dstEntry.Attributes.FileSize {
|
||||
dstEntry.Attributes.Md5 = util.Md5(dstEntry.Content)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Use unified copy strategy approach
|
||||
dstChunks, dstMetadata, copyErr := s3a.executeUnifiedCopyStrategy(entry, r, dstBucket, srcObject, dstObject)
|
||||
dstChunks, dstMetadata, copyErr := s3a.executeUnifiedCopyStrategy(entry, r, srcBucket, dstBucket, srcObject, dstObject)
|
||||
if copyErr != nil {
|
||||
glog.Errorf("CopyObjectHandler unified copy error: %v", copyErr)
|
||||
// Map errors to appropriate S3 errors
|
||||
@@ -257,6 +279,10 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
glog.V(2).Infof("Applied %d destination metadata entries for copy: %s", len(dstMetadata), r.URL.Path)
|
||||
}
|
||||
|
||||
if dstEntry.Attributes != nil && len(dstEntry.Attributes.Md5) == 0 && canReuseSourceMd5 {
|
||||
dstEntry.Attributes.Md5 = append([]byte(nil), sourceMd5...)
|
||||
}
|
||||
}
|
||||
|
||||
// Check if destination bucket has versioning enabled
|
||||
|
||||
@@ -13,26 +13,16 @@ import (
|
||||
|
||||
// executeUnifiedCopyStrategy executes the appropriate copy strategy based on encryption state
|
||||
// Returns chunks and destination metadata that should be applied to the destination entry
|
||||
func (s3a *S3ApiServer) executeUnifiedCopyStrategy(entry *filer_pb.Entry, r *http.Request, dstBucket, srcObject, dstObject string) ([]*filer_pb.FileChunk, map[string][]byte, error) {
|
||||
func (s3a *S3ApiServer) executeUnifiedCopyStrategy(entry *filer_pb.Entry, r *http.Request, srcBucket, dstBucket, srcObject, dstObject string) ([]*filer_pb.FileChunk, map[string][]byte, error) {
|
||||
// Detect encryption state (using entry-aware detection for multipart objects)
|
||||
srcPath := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, r.Header.Get("X-Amz-Copy-Source-Bucket"), srcObject)
|
||||
srcPath := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, srcBucket, srcObject)
|
||||
dstPath := fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, dstBucket, dstObject)
|
||||
state := DetectEncryptionStateWithEntry(entry, r, srcPath, dstPath)
|
||||
|
||||
// Debug logging for encryption state
|
||||
|
||||
// Apply bucket default encryption if no explicit encryption specified
|
||||
if !state.IsTargetEncrypted() {
|
||||
bucketMetadata, err := s3a.getBucketMetadata(dstBucket)
|
||||
if err == nil && bucketMetadata != nil && bucketMetadata.Encryption != nil {
|
||||
switch bucketMetadata.Encryption.SseAlgorithm {
|
||||
case "aws:kms":
|
||||
state.DstSSEKMS = true
|
||||
case "AES256":
|
||||
state.DstSSES3 = true
|
||||
}
|
||||
}
|
||||
}
|
||||
s3a.applyCopyBucketDefaultEncryption(state, dstBucket)
|
||||
|
||||
// Determine copy strategy
|
||||
strategy, err := DetermineUnifiedCopyStrategy(state, entry.Extended, r)
|
||||
@@ -169,3 +159,18 @@ func (s3a *S3ApiServer) executeReencryptCopy(entry *filer_pb.Entry, r *http.Requ
|
||||
glog.V(2).Infof("Cross-encryption copy: using unified multipart copy")
|
||||
return s3a.copyMultipartCrossEncryption(entry, r, state, dstBucket, dstPath)
|
||||
}
|
||||
|
||||
// applyCopyBucketDefaultEncryption applies the destination bucket's default encryption settings if no explicit encryption is specified
|
||||
func (s3a *S3ApiServer) applyCopyBucketDefaultEncryption(state *EncryptionState, dstBucket string) {
|
||||
if !state.IsTargetEncrypted() {
|
||||
bucketMetadata, err := s3a.getBucketMetadata(dstBucket)
|
||||
if err == nil && bucketMetadata != nil && bucketMetadata.Encryption != nil {
|
||||
switch bucketMetadata.Encryption.SseAlgorithm {
|
||||
case "aws:kms":
|
||||
state.DstSSEKMS = true
|
||||
case "AES256":
|
||||
state.DstSSES3 = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -428,6 +428,11 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
|
||||
// API Router
|
||||
apiRouter := router.PathPrefix("/").Subrouter()
|
||||
|
||||
// S3 Tables API endpoint
|
||||
// POST / with X-Amz-Target: S3Tables.<OperationName>
|
||||
// plus REST-style endpoints for AWS CLI
|
||||
s3a.registerS3TablesRoutes(apiRouter)
|
||||
|
||||
// Readiness Probe
|
||||
apiRouter.Methods(http.MethodGet).Path("/status").HandlerFunc(s3a.StatusHandler)
|
||||
apiRouter.Methods(http.MethodGet).Path("/healthz").HandlerFunc(s3a.StatusHandler)
|
||||
|
||||
@@ -0,0 +1,656 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
|
||||
)
|
||||
|
||||
// S3TablesApiServer wraps the S3 Tables handler with S3ApiServer's filer access
|
||||
type S3TablesApiServer struct {
|
||||
s3a *S3ApiServer
|
||||
handler *s3tables.S3TablesHandler
|
||||
}
|
||||
|
||||
// NewS3TablesApiServer creates a new S3 Tables API server
|
||||
func NewS3TablesApiServer(s3a *S3ApiServer) *S3TablesApiServer {
|
||||
return &S3TablesApiServer{
|
||||
s3a: s3a,
|
||||
handler: s3tables.NewS3TablesHandler(),
|
||||
}
|
||||
}
|
||||
|
||||
// SetRegion sets the AWS region for ARN generation
|
||||
func (st *S3TablesApiServer) SetRegion(region string) {
|
||||
st.handler.SetRegion(region)
|
||||
}
|
||||
|
||||
// SetAccountID sets the AWS account ID for ARN generation
|
||||
func (st *S3TablesApiServer) SetAccountID(accountID string) {
|
||||
st.handler.SetAccountID(accountID)
|
||||
}
|
||||
|
||||
// S3TablesHandler handles S3 Tables API requests
|
||||
func (st *S3TablesApiServer) S3TablesHandler(w http.ResponseWriter, r *http.Request) {
|
||||
st.handler.HandleRequest(w, r, st)
|
||||
}
|
||||
|
||||
// WithFilerClient implements the s3tables.FilerClient interface
|
||||
func (st *S3TablesApiServer) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||
return st.s3a.WithFilerClient(streamingMode, fn)
|
||||
}
|
||||
|
||||
// registerS3TablesRoutes registers S3 Tables API routes
|
||||
func (s3a *S3ApiServer) registerS3TablesRoutes(router *mux.Router) {
|
||||
// Create S3 Tables handler
|
||||
s3TablesApi := NewS3TablesApiServer(s3a)
|
||||
|
||||
// Regex for S3 Tables Bucket ARN
|
||||
const tableBucketARNRegex = "arn:aws:s3tables:[^/:]*:[^/:]*:bucket/[^/]+"
|
||||
|
||||
// REST-style S3 Tables API routes (used by AWS CLI)
|
||||
targetMatcher := func(r *http.Request, rm *mux.RouteMatch) bool {
|
||||
return strings.HasPrefix(r.Header.Get("X-Amz-Target"), "S3Tables.")
|
||||
}
|
||||
router.Methods(http.MethodPost).Path("/").MatcherFunc(targetMatcher).
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.S3TablesHandler), "S3Tables-Target"))
|
||||
router.Methods(http.MethodPut).Path("/buckets").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("CreateTableBucket", buildCreateTableBucketRequest)), "S3Tables-CreateTableBucket"))
|
||||
router.Methods(http.MethodGet).Path("/buckets").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("ListTableBuckets", buildListTableBucketsRequest)), "S3Tables-ListTableBuckets"))
|
||||
router.Methods(http.MethodGet).Path("/buckets/{tableBucketARN:" + tableBucketARNRegex + "}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("GetTableBucket", buildTableBucketArnRequest)), "S3Tables-GetTableBucket"))
|
||||
router.Methods(http.MethodDelete).Path("/buckets/{tableBucketARN:" + tableBucketARNRegex + "}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("DeleteTableBucket", buildDeleteTableBucketRequest)), "S3Tables-DeleteTableBucket"))
|
||||
router.Methods(http.MethodPut).Path("/buckets/{tableBucketARN:" + tableBucketARNRegex + "}/policy").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("PutTableBucketPolicy", buildPutTableBucketPolicyRequest)), "S3Tables-PutTableBucketPolicy"))
|
||||
router.Methods(http.MethodGet).Path("/buckets/{tableBucketARN:" + tableBucketARNRegex + "}/policy").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("GetTableBucketPolicy", buildGetTableBucketPolicyRequest)), "S3Tables-GetTableBucketPolicy"))
|
||||
router.Methods(http.MethodDelete).Path("/buckets/{tableBucketARN:" + tableBucketARNRegex + "}/policy").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("DeleteTableBucketPolicy", buildDeleteTableBucketPolicyRequest)), "S3Tables-DeleteTableBucketPolicy"))
|
||||
|
||||
router.Methods(http.MethodPut).Path("/namespaces/{tableBucketARN:" + tableBucketARNRegex + "}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("CreateNamespace", buildCreateNamespaceRequest)), "S3Tables-CreateNamespace"))
|
||||
router.Methods(http.MethodGet).Path("/namespaces/{tableBucketARN:" + tableBucketARNRegex + "}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("ListNamespaces", buildListNamespacesRequest)), "S3Tables-ListNamespaces"))
|
||||
router.Methods(http.MethodGet).Path("/namespaces/{tableBucketARN:" + tableBucketARNRegex + "}/{namespace}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("GetNamespace", buildGetNamespaceRequest)), "S3Tables-GetNamespace"))
|
||||
router.Methods(http.MethodDelete).Path("/namespaces/{tableBucketARN:" + tableBucketARNRegex + "}/{namespace}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("DeleteNamespace", buildDeleteNamespaceRequest)), "S3Tables-DeleteNamespace"))
|
||||
|
||||
router.Methods(http.MethodPut).Path("/tables/{tableBucketARN:" + tableBucketARNRegex + "}/{namespace}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("CreateTable", buildCreateTableRequest)), "S3Tables-CreateTable"))
|
||||
router.Methods(http.MethodGet).Path("/tables/{tableBucketARN:" + tableBucketARNRegex + "}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("ListTables", buildListTablesRequest)), "S3Tables-ListTables"))
|
||||
router.Methods(http.MethodDelete).Path("/tables/{tableBucketARN:" + tableBucketARNRegex + "}/{namespace}/{name}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("DeleteTable", buildDeleteTableRequest)), "S3Tables-DeleteTable"))
|
||||
|
||||
router.Methods(http.MethodPut).Path("/tables/{tableBucketARN:" + tableBucketARNRegex + "}/{namespace}/{name}/policy").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("PutTablePolicy", buildPutTablePolicyRequest)), "S3Tables-PutTablePolicy"))
|
||||
router.Methods(http.MethodGet).Path("/tables/{tableBucketARN:" + tableBucketARNRegex + "}/{namespace}/{name}/policy").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("GetTablePolicy", buildGetTablePolicyRequest)), "S3Tables-GetTablePolicy"))
|
||||
router.Methods(http.MethodDelete).Path("/tables/{tableBucketARN:" + tableBucketARNRegex + "}/{namespace}/{name}/policy").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("DeleteTablePolicy", buildDeleteTablePolicyRequest)), "S3Tables-DeleteTablePolicy"))
|
||||
|
||||
router.Methods(http.MethodPost).Path("/tag/{resourceArn:arn:aws:s3tables:.*}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("TagResource", buildTagResourceRequest)), "S3Tables-TagResource"))
|
||||
router.Methods(http.MethodGet).Path("/tag/{resourceArn:arn:aws:s3tables:.*}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("ListTagsForResource", buildListTagsForResourceRequest)), "S3Tables-ListTagsForResource"))
|
||||
router.Methods(http.MethodDelete).Path("/tag/{resourceArn:arn:aws:s3tables:.*}").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("UntagResource", buildUntagResourceRequest)), "S3Tables-UntagResource"))
|
||||
|
||||
router.Methods(http.MethodGet).Path("/get-table").
|
||||
HandlerFunc(track(s3a.authenticateS3Tables(s3TablesApi.handleRestOperation("GetTable", buildGetTableRequest)), "S3Tables-GetTable"))
|
||||
|
||||
glog.V(1).Infof("S3 Tables API enabled")
|
||||
}
|
||||
|
||||
type s3tablesRequestBuilder func(r *http.Request) (interface{}, error)
|
||||
|
||||
func (st *S3TablesApiServer) handleRestOperation(operation string, builder s3tablesRequestBuilder) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
payload, err := builder(r)
|
||||
if err != nil {
|
||||
writeS3TablesError(w, http.StatusBadRequest, s3tables.ErrCodeInvalidRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if err := setS3TablesRequestBody(r, payload); err != nil {
|
||||
writeS3TablesError(w, http.StatusInternalServerError, s3tables.ErrCodeInternalError, err.Error())
|
||||
return
|
||||
}
|
||||
r.Header.Set("X-Amz-Target", "S3Tables."+operation)
|
||||
st.S3TablesHandler(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
func setS3TablesRequestBody(r *http.Request, payload interface{}) error {
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
||||
r.ContentLength = int64(len(body))
|
||||
r.Header.Set("Content-Type", "application/x-amz-json-1.1")
|
||||
return nil
|
||||
}
|
||||
|
||||
func readS3TablesJSONBody(r *http.Request, v interface{}) error {
|
||||
if r.Body == nil {
|
||||
return nil
|
||||
}
|
||||
defer r.Body.Close()
|
||||
const maxRequestBodySize = 10 * 1024 * 1024
|
||||
if r.ContentLength > maxRequestBodySize {
|
||||
return fmt.Errorf("request body too large: exceeds maximum size of %d bytes", maxRequestBodySize)
|
||||
}
|
||||
limitedReader := io.LimitReader(r.Body, maxRequestBodySize+1)
|
||||
body, err := io.ReadAll(limitedReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(body) > maxRequestBodySize {
|
||||
return fmt.Errorf("request body too large: exceeds maximum size of %d bytes", maxRequestBodySize)
|
||||
}
|
||||
if len(bytes.TrimSpace(body)) == 0 {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(body, v)
|
||||
}
|
||||
|
||||
func writeS3TablesError(w http.ResponseWriter, status int, code, message string) {
|
||||
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
|
||||
w.WriteHeader(status)
|
||||
errorResponse := map[string]interface{}{
|
||||
"__type": code,
|
||||
"message": message,
|
||||
}
|
||||
if err := json.NewEncoder(w).Encode(errorResponse); err != nil {
|
||||
glog.Errorf("failed to encode S3Tables error response (status=%d, code=%s, message=%q): %v", status, code, message, err)
|
||||
}
|
||||
}
|
||||
|
||||
func getDecodedPathParam(r *http.Request, name string) (string, error) {
|
||||
value := mux.Vars(r)[name]
|
||||
if value == "" {
|
||||
return "", nil
|
||||
}
|
||||
decoded, err := url.PathUnescape(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if decoded == ".." || strings.Contains(decoded, "../") || strings.Contains(decoded, `..\`) || strings.Contains(decoded, "\x00") {
|
||||
return "", fmt.Errorf("invalid path parameter %s", name)
|
||||
}
|
||||
return decoded, nil
|
||||
}
|
||||
|
||||
func buildTableBucketRequestWithARN(r *http.Request, constructor func(string) interface{}) (interface{}, error) {
|
||||
arn, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if arn == "" {
|
||||
return nil, fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
if _, err := s3tables.ParseBucketNameFromARN(arn); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return constructor(arn), nil
|
||||
}
|
||||
|
||||
func parseOptionalIntParam(r *http.Request, name string) (int, error) {
|
||||
value := r.URL.Query().Get(name)
|
||||
if value == "" {
|
||||
return 0, nil
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%s must be an integer", name)
|
||||
}
|
||||
if parsed <= 0 {
|
||||
return 0, fmt.Errorf("%s must be a positive integer", name)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func parseOptionalNamespace(r *http.Request, name string) []string {
|
||||
value := r.URL.Query().Get(name)
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := s3tables.ValidateNamespace([]string{value}); err != nil {
|
||||
glog.V(1).Infof("invalid namespace value for %s: %q: %v", name, value, err)
|
||||
return nil
|
||||
}
|
||||
return []string{value}
|
||||
}
|
||||
|
||||
// parseTagKeys handles tag key parsing from query parameters.
|
||||
// If a single value contains commas, it is split into multiple keys (e.g., "key1,key2,key3").
|
||||
// Otherwise, multiple query values are returned as-is.
|
||||
func parseTagKeys(values []string) []string {
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
for _, part := range strings.Split(value, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part != "" {
|
||||
out = append(out, part)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func buildCreateTableBucketRequest(r *http.Request) (interface{}, error) {
|
||||
var req s3tables.CreateTableBucketRequest
|
||||
if err := readS3TablesJSONBody(r, &req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
func buildListTableBucketsRequest(r *http.Request) (interface{}, error) {
|
||||
maxBuckets, err := parseOptionalIntParam(r, "maxBuckets")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3tables.ListTableBucketsRequest{
|
||||
Prefix: r.URL.Query().Get("prefix"),
|
||||
ContinuationToken: r.URL.Query().Get("continuationToken"),
|
||||
MaxBuckets: maxBuckets,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildTableBucketArnRequest(r *http.Request) (interface{}, error) {
|
||||
return buildTableBucketRequestWithARN(r, func(arn string) interface{} {
|
||||
return &s3tables.GetTableBucketRequest{TableBucketARN: arn}
|
||||
})
|
||||
}
|
||||
|
||||
func buildDeleteTableBucketRequest(r *http.Request) (interface{}, error) {
|
||||
return buildTableBucketRequestWithARN(r, func(arn string) interface{} {
|
||||
return &s3tables.DeleteTableBucketRequest{TableBucketARN: arn}
|
||||
})
|
||||
}
|
||||
|
||||
func buildPutTableBucketPolicyRequest(r *http.Request) (interface{}, error) {
|
||||
var req s3tables.PutTableBucketPolicyRequest
|
||||
if err := readS3TablesJSONBody(r, &req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.TableBucketARN = tableBucketARN
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
func buildGetTableBucketPolicyRequest(r *http.Request) (interface{}, error) {
|
||||
return buildTableBucketRequestWithARN(r, func(arn string) interface{} {
|
||||
return &s3tables.GetTableBucketPolicyRequest{TableBucketARN: arn}
|
||||
})
|
||||
}
|
||||
|
||||
func buildDeleteTableBucketPolicyRequest(r *http.Request) (interface{}, error) {
|
||||
return buildTableBucketRequestWithARN(r, func(arn string) interface{} {
|
||||
return &s3tables.DeleteTableBucketPolicyRequest{TableBucketARN: arn}
|
||||
})
|
||||
}
|
||||
|
||||
func buildCreateNamespaceRequest(r *http.Request) (interface{}, error) {
|
||||
var req s3tables.CreateNamespaceRequest
|
||||
if err := readS3TablesJSONBody(r, &req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.TableBucketARN = tableBucketARN
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
func buildListNamespacesRequest(r *http.Request) (interface{}, error) {
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maxNamespaces, err := parseOptionalIntParam(r, "maxNamespaces")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3tables.ListNamespacesRequest{
|
||||
TableBucketARN: tableBucketARN,
|
||||
Prefix: r.URL.Query().Get("prefix"),
|
||||
ContinuationToken: r.URL.Query().Get("continuationToken"),
|
||||
MaxNamespaces: maxNamespaces,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildGetNamespaceRequest(r *http.Request) (interface{}, error) {
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
namespace, err := getDecodedPathParam(r, "namespace")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if namespace == "" {
|
||||
return nil, fmt.Errorf("namespace is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3tables.GetNamespaceRequest{
|
||||
TableBucketARN: tableBucketARN,
|
||||
Namespace: []string{namespace},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildDeleteNamespaceRequest(r *http.Request) (interface{}, error) {
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
namespace, err := getDecodedPathParam(r, "namespace")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if namespace == "" {
|
||||
return nil, fmt.Errorf("namespace is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3tables.DeleteNamespaceRequest{
|
||||
TableBucketARN: tableBucketARN,
|
||||
Namespace: []string{namespace},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildCreateTableRequest(r *http.Request) (interface{}, error) {
|
||||
var req s3tables.CreateTableRequest
|
||||
if err := readS3TablesJSONBody(r, &req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
namespace, err := getDecodedPathParam(r, "namespace")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if namespace == "" {
|
||||
return nil, fmt.Errorf("namespace is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.TableBucketARN = tableBucketARN
|
||||
req.Namespace = []string{namespace}
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
func buildListTablesRequest(r *http.Request) (interface{}, error) {
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maxTables, err := parseOptionalIntParam(r, "maxTables")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3tables.ListTablesRequest{
|
||||
TableBucketARN: tableBucketARN,
|
||||
Namespace: parseOptionalNamespace(r, "namespace"),
|
||||
Prefix: r.URL.Query().Get("prefix"),
|
||||
ContinuationToken: r.URL.Query().Get("continuationToken"),
|
||||
MaxTables: maxTables,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildGetTableRequest(r *http.Request) (interface{}, error) {
|
||||
query := r.URL.Query()
|
||||
tableARN := query.Get("tableArn")
|
||||
req := &s3tables.GetTableRequest{
|
||||
TableARN: tableARN,
|
||||
}
|
||||
if tableARN == "" {
|
||||
req.TableBucketARN = query.Get("tableBucketARN")
|
||||
req.Namespace = parseOptionalNamespace(r, "namespace")
|
||||
req.Name = query.Get("name")
|
||||
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
|
||||
return nil, fmt.Errorf("either tableArn or (tableBucketARN, namespace, name) must be provided")
|
||||
}
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func buildDeleteTableRequest(r *http.Request) (interface{}, error) {
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
namespace, err := getDecodedPathParam(r, "namespace")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if namespace == "" {
|
||||
return nil, fmt.Errorf("namespace is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name, err := getDecodedPathParam(r, "name")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("name is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateTableName(name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3tables.DeleteTableRequest{
|
||||
TableBucketARN: tableBucketARN,
|
||||
Namespace: []string{namespace},
|
||||
Name: name,
|
||||
VersionToken: r.URL.Query().Get("versionToken"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildPutTablePolicyRequest(r *http.Request) (interface{}, error) {
|
||||
var req s3tables.PutTablePolicyRequest
|
||||
if err := readS3TablesJSONBody(r, &req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
namespace, err := getDecodedPathParam(r, "namespace")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if namespace == "" {
|
||||
return nil, fmt.Errorf("namespace is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name, err := getDecodedPathParam(r, "name")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("name is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateTableName(name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.TableBucketARN = tableBucketARN
|
||||
req.Namespace = []string{namespace}
|
||||
req.Name = name
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
func buildGetTablePolicyRequest(r *http.Request) (interface{}, error) {
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
namespace, err := getDecodedPathParam(r, "namespace")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if namespace == "" {
|
||||
return nil, fmt.Errorf("namespace is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name, err := getDecodedPathParam(r, "name")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("name is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateTableName(name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3tables.GetTablePolicyRequest{
|
||||
TableBucketARN: tableBucketARN,
|
||||
Namespace: []string{namespace},
|
||||
Name: name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildDeleteTablePolicyRequest(r *http.Request) (interface{}, error) {
|
||||
tableBucketARN, err := getDecodedPathParam(r, "tableBucketARN")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
namespace, err := getDecodedPathParam(r, "namespace")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if namespace == "" {
|
||||
return nil, fmt.Errorf("namespace is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateNamespace([]string{namespace}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name, err := getDecodedPathParam(r, "name")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("name is required")
|
||||
}
|
||||
if _, err := s3tables.ValidateTableName(name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &s3tables.DeleteTablePolicyRequest{
|
||||
TableBucketARN: tableBucketARN,
|
||||
Namespace: []string{namespace},
|
||||
Name: name,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildTagResourceRequest(r *http.Request) (interface{}, error) {
|
||||
var req s3tables.TagResourceRequest
|
||||
if err := readS3TablesJSONBody(r, &req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resourceARN, err := getDecodedPathParam(r, "resourceArn")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resourceARN == "" {
|
||||
return nil, fmt.Errorf("resourceArn is required")
|
||||
}
|
||||
req.ResourceARN = resourceARN
|
||||
return &req, nil
|
||||
}
|
||||
|
||||
func buildListTagsForResourceRequest(r *http.Request) (interface{}, error) {
|
||||
resourceARN, err := getDecodedPathParam(r, "resourceArn")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resourceARN == "" {
|
||||
return nil, fmt.Errorf("resourceArn is required")
|
||||
}
|
||||
return &s3tables.ListTagsForResourceRequest{
|
||||
ResourceARN: resourceARN,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildUntagResourceRequest(r *http.Request) (interface{}, error) {
|
||||
resourceARN, err := getDecodedPathParam(r, "resourceArn")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resourceARN == "" {
|
||||
return nil, fmt.Errorf("resourceArn is required")
|
||||
}
|
||||
tagKeys := parseTagKeys(r.URL.Query()["tagKeys"])
|
||||
if len(tagKeys) == 0 {
|
||||
return nil, fmt.Errorf("tagKeys is required for %s", resourceARN)
|
||||
}
|
||||
return &s3tables.UntagResourceRequest{
|
||||
ResourceARN: resourceARN,
|
||||
TagKeys: tagKeys,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// authenticateS3Tables wraps the handler with IAM authentication using AuthSignatureOnly
|
||||
// This authenticates the request but delegates authorization to the S3 Tables handler
|
||||
// which performs granular permission checks based on the specific operation.
|
||||
func (s3a *S3ApiServer) authenticateS3Tables(f http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if !s3a.iam.isEnabled() {
|
||||
f(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// Use AuthSignatureOnly to authenticate the request without authorizing specific actions
|
||||
identity, errCode := s3a.iam.AuthSignatureOnly(r)
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Store the authenticated identity in request context
|
||||
if identity != nil && identity.Name != "" {
|
||||
ctx := s3_constants.SetIdentityNameInContext(r.Context(), identity.Name)
|
||||
ctx = s3_constants.SetIdentityInContext(ctx, identity)
|
||||
r = r.WithContext(ctx)
|
||||
}
|
||||
|
||||
f(w, r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrAttributeNotFound = errors.New("attribute not found")
|
||||
)
|
||||
|
||||
// Filer operations - Common functions for interacting with the filer
|
||||
|
||||
// createDirectory creates a new directory at the specified path
|
||||
func (h *S3TablesHandler) createDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
|
||||
dir, name := splitPath(path)
|
||||
now := time.Now().Unix()
|
||||
_, err := client.CreateEntry(ctx, &filer_pb.CreateEntryRequest{
|
||||
Directory: dir,
|
||||
Entry: &filer_pb.Entry{
|
||||
Name: name,
|
||||
IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{
|
||||
Mtime: now,
|
||||
Crtime: now,
|
||||
FileMode: uint32(0755 | os.ModeDir), // Directory mode
|
||||
},
|
||||
},
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// setExtendedAttribute sets an extended attribute on an existing entry
|
||||
func (h *S3TablesHandler) setExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string, data []byte) error {
|
||||
dir, name := splitPath(path)
|
||||
|
||||
// First, get the existing entry
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
entry := resp.Entry
|
||||
|
||||
// Update the extended attributes
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
entry.Extended[key] = data
|
||||
|
||||
// Save the updated entry
|
||||
_, err = client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
|
||||
Directory: dir,
|
||||
Entry: entry,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// getExtendedAttribute gets an extended attribute from an entry
|
||||
func (h *S3TablesHandler) getExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string) ([]byte, error) {
|
||||
dir, name := splitPath(path)
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.Entry.Extended == nil {
|
||||
return nil, fmt.Errorf("%w: %s", ErrAttributeNotFound, key)
|
||||
}
|
||||
|
||||
data, ok := resp.Entry.Extended[key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: %s", ErrAttributeNotFound, key)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// deleteExtendedAttribute deletes an extended attribute from an entry
|
||||
func (h *S3TablesHandler) deleteExtendedAttribute(ctx context.Context, client filer_pb.SeaweedFilerClient, path, key string) error {
|
||||
dir, name := splitPath(path)
|
||||
|
||||
// Get the existing entry
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
entry := resp.Entry
|
||||
|
||||
// Remove the extended attribute
|
||||
if entry.Extended != nil {
|
||||
delete(entry.Extended, key)
|
||||
}
|
||||
|
||||
// Save the updated entry
|
||||
_, err = client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
|
||||
Directory: dir,
|
||||
Entry: entry,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// deleteDirectory deletes a directory and all its contents
|
||||
func (h *S3TablesHandler) deleteDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) error {
|
||||
dir, name := splitPath(path)
|
||||
_, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
IsDeleteData: true,
|
||||
IsRecursive: true,
|
||||
IgnoreRecursiveError: true,
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// entryExists checks if an entry exists at the given path
|
||||
func (h *S3TablesHandler) entryExists(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) bool {
|
||||
dir, name := splitPath(path)
|
||||
_, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
return err == nil
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
const (
|
||||
TablesPath = "/table-buckets"
|
||||
DefaultAccountID = "000000000000"
|
||||
DefaultRegion = "us-east-1"
|
||||
|
||||
// Extended entry attributes for metadata storage
|
||||
ExtendedKeyMetadata = "s3tables.metadata"
|
||||
ExtendedKeyPolicy = "s3tables.policy"
|
||||
ExtendedKeyTags = "s3tables.tags"
|
||||
|
||||
// Maximum request body size (10MB)
|
||||
maxRequestBodySize = 10 * 1024 * 1024
|
||||
)
|
||||
|
||||
var (
|
||||
ErrVersionTokenMismatch = errors.New("version token mismatch")
|
||||
ErrAccessDenied = errors.New("access denied")
|
||||
)
|
||||
|
||||
type ResourceType string
|
||||
|
||||
const (
|
||||
ResourceTypeBucket ResourceType = "bucket"
|
||||
ResourceTypeTable ResourceType = "table"
|
||||
)
|
||||
|
||||
// S3TablesHandler handles S3 Tables API requests
|
||||
type S3TablesHandler struct {
|
||||
region string
|
||||
accountID string
|
||||
}
|
||||
|
||||
// NewS3TablesHandler creates a new S3 Tables handler
|
||||
func NewS3TablesHandler() *S3TablesHandler {
|
||||
return &S3TablesHandler{
|
||||
region: DefaultRegion,
|
||||
accountID: DefaultAccountID,
|
||||
}
|
||||
}
|
||||
|
||||
// SetRegion sets the AWS region for ARN generation
|
||||
func (h *S3TablesHandler) SetRegion(region string) {
|
||||
if region != "" {
|
||||
h.region = region
|
||||
}
|
||||
}
|
||||
|
||||
// SetAccountID sets the AWS account ID for ARN generation
|
||||
func (h *S3TablesHandler) SetAccountID(accountID string) {
|
||||
if accountID != "" {
|
||||
h.accountID = accountID
|
||||
}
|
||||
}
|
||||
|
||||
// FilerClient interface for filer operations
|
||||
type FilerClient interface {
|
||||
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
|
||||
}
|
||||
|
||||
// HandleRequest is the main entry point for S3 Tables API requests
|
||||
func (h *S3TablesHandler) HandleRequest(w http.ResponseWriter, r *http.Request, filerClient FilerClient) {
|
||||
operation := r.Header.Get("X-Amz-Target")
|
||||
if operation != "" {
|
||||
if idx := strings.LastIndex(operation, "."); idx != -1 {
|
||||
operation = operation[idx+1:]
|
||||
}
|
||||
}
|
||||
if operation == "" {
|
||||
glog.V(1).Infof("S3Tables: missing X-Amz-Target header")
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "Missing X-Amz-Target header")
|
||||
return
|
||||
}
|
||||
|
||||
glog.V(3).Infof("S3Tables: handling operation %s", operation)
|
||||
|
||||
var err error
|
||||
switch operation {
|
||||
// Table Bucket operations
|
||||
case "CreateTableBucket":
|
||||
err = h.handleCreateTableBucket(w, r, filerClient)
|
||||
case "GetTableBucket":
|
||||
err = h.handleGetTableBucket(w, r, filerClient)
|
||||
case "ListTableBuckets":
|
||||
err = h.handleListTableBuckets(w, r, filerClient)
|
||||
case "DeleteTableBucket":
|
||||
err = h.handleDeleteTableBucket(w, r, filerClient)
|
||||
|
||||
// Table Bucket Policy operations
|
||||
case "PutTableBucketPolicy":
|
||||
err = h.handlePutTableBucketPolicy(w, r, filerClient)
|
||||
case "GetTableBucketPolicy":
|
||||
err = h.handleGetTableBucketPolicy(w, r, filerClient)
|
||||
case "DeleteTableBucketPolicy":
|
||||
err = h.handleDeleteTableBucketPolicy(w, r, filerClient)
|
||||
|
||||
// Namespace operations
|
||||
case "CreateNamespace":
|
||||
err = h.handleCreateNamespace(w, r, filerClient)
|
||||
case "GetNamespace":
|
||||
err = h.handleGetNamespace(w, r, filerClient)
|
||||
case "ListNamespaces":
|
||||
err = h.handleListNamespaces(w, r, filerClient)
|
||||
case "DeleteNamespace":
|
||||
err = h.handleDeleteNamespace(w, r, filerClient)
|
||||
|
||||
// Table operations
|
||||
case "CreateTable":
|
||||
err = h.handleCreateTable(w, r, filerClient)
|
||||
case "GetTable":
|
||||
err = h.handleGetTable(w, r, filerClient)
|
||||
case "ListTables":
|
||||
err = h.handleListTables(w, r, filerClient)
|
||||
case "DeleteTable":
|
||||
err = h.handleDeleteTable(w, r, filerClient)
|
||||
|
||||
// Table Policy operations
|
||||
case "PutTablePolicy":
|
||||
err = h.handlePutTablePolicy(w, r, filerClient)
|
||||
case "GetTablePolicy":
|
||||
err = h.handleGetTablePolicy(w, r, filerClient)
|
||||
case "DeleteTablePolicy":
|
||||
err = h.handleDeleteTablePolicy(w, r, filerClient)
|
||||
|
||||
// Tagging operations
|
||||
case "TagResource":
|
||||
err = h.handleTagResource(w, r, filerClient)
|
||||
case "ListTagsForResource":
|
||||
err = h.handleListTagsForResource(w, r, filerClient)
|
||||
case "UntagResource":
|
||||
err = h.handleUntagResource(w, r, filerClient)
|
||||
|
||||
default:
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, fmt.Sprintf("Unknown operation: %s", operation))
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("S3Tables: error handling %s: %v", operation, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Principal/authorization helpers
|
||||
|
||||
// getAccountID returns the authenticated account ID from the request or the handler's default.
|
||||
// This is also used as the principal for permission checks, ensuring alignment between
|
||||
// the caller identity and ownership verification when IAM is enabled.
|
||||
func (h *S3TablesHandler) getAccountID(r *http.Request) string {
|
||||
if identityName := s3_constants.GetIdentityNameFromContext(r); identityName != "" {
|
||||
return identityName
|
||||
}
|
||||
if accountID := r.Header.Get(s3_constants.AmzAccountId); accountID != "" {
|
||||
return accountID
|
||||
}
|
||||
return h.accountID
|
||||
}
|
||||
|
||||
// getIdentityActions extracts the action list from the identity object in the request context.
|
||||
// Uses reflection to avoid import cycles with s3api package.
|
||||
func getIdentityActions(r *http.Request) []string {
|
||||
identityRaw := s3_constants.GetIdentityFromContext(r)
|
||||
if identityRaw == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Use reflection to access the Actions field to avoid import cycle
|
||||
val := reflect.ValueOf(identityRaw)
|
||||
if val.Kind() == reflect.Ptr {
|
||||
val = val.Elem()
|
||||
}
|
||||
if val.Kind() != reflect.Struct {
|
||||
return nil
|
||||
}
|
||||
|
||||
actionsField := val.FieldByName("Actions")
|
||||
if !actionsField.IsValid() || actionsField.Kind() != reflect.Slice {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert actions to string slice
|
||||
actions := make([]string, actionsField.Len())
|
||||
for i := 0; i < actionsField.Len(); i++ {
|
||||
action := actionsField.Index(i)
|
||||
// Action is likely a custom type (e.g., type Action string)
|
||||
// Convert to string using String() or direct string conversion
|
||||
if action.Kind() == reflect.String {
|
||||
actions[i] = action.String()
|
||||
} else if action.CanInterface() {
|
||||
// Try to convert via fmt.Sprint
|
||||
actions[i] = fmt.Sprint(action.Interface())
|
||||
}
|
||||
}
|
||||
return actions
|
||||
}
|
||||
|
||||
// Request/Response helpers
|
||||
|
||||
func (h *S3TablesHandler) readRequestBody(r *http.Request, v interface{}) error {
|
||||
defer r.Body.Close()
|
||||
|
||||
// Limit request body size to prevent unbounded reads
|
||||
limitedReader := io.LimitReader(r.Body, maxRequestBodySize+1)
|
||||
body, err := io.ReadAll(limitedReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read request body: %w", err)
|
||||
}
|
||||
|
||||
// Check if body exceeds size limit
|
||||
if len(body) > maxRequestBodySize {
|
||||
return fmt.Errorf("request body too large: exceeds maximum size of %d bytes", maxRequestBodySize)
|
||||
}
|
||||
|
||||
if len(body) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, v); err != nil {
|
||||
return fmt.Errorf("failed to decode request: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Response writing helpers
|
||||
|
||||
func (h *S3TablesHandler) writeJSON(w http.ResponseWriter, status int, data interface{}) {
|
||||
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
|
||||
w.WriteHeader(status)
|
||||
if data != nil {
|
||||
if err := json.NewEncoder(w).Encode(data); err != nil {
|
||||
glog.Errorf("S3Tables: failed to encode response: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) writeError(w http.ResponseWriter, status int, code, message string) {
|
||||
w.Header().Set("Content-Type", "application/x-amz-json-1.1")
|
||||
w.WriteHeader(status)
|
||||
errorResponse := map[string]interface{}{
|
||||
"__type": code,
|
||||
"message": message,
|
||||
}
|
||||
if err := json.NewEncoder(w).Encode(errorResponse); err != nil {
|
||||
glog.Errorf("S3Tables: failed to encode error response: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ARN generation helpers
|
||||
|
||||
func (h *S3TablesHandler) generateTableBucketARN(ownerAccountID, bucketName string) string {
|
||||
return fmt.Sprintf("arn:aws:s3tables:%s:%s:bucket/%s", h.region, ownerAccountID, bucketName)
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) generateTableARN(ownerAccountID, bucketName, tableID string) string {
|
||||
return fmt.Sprintf("arn:aws:s3tables:%s:%s:bucket/%s/table/%s", h.region, ownerAccountID, bucketName, tableID)
|
||||
}
|
||||
|
||||
func isAuthError(err error) bool {
|
||||
var authErr *AuthError
|
||||
return errors.As(err, &authErr) || errors.Is(err, ErrAccessDenied)
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) readTags(ctx context.Context, client filer_pb.SeaweedFilerClient, path string) (map[string]string, error) {
|
||||
data, err := h.getExtendedAttribute(ctx, client, path, ExtendedKeyTags)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
tags := make(map[string]string)
|
||||
if err := json.Unmarshal(data, &tags); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal tags: %w", err)
|
||||
}
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
func mapKeys(tags map[string]string) []string {
|
||||
if len(tags) == 0 {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(tags))
|
||||
for key := range tags {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// handleCreateTableBucket creates a new table bucket
|
||||
func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
// Check permission
|
||||
principal := h.getAccountID(r)
|
||||
if !CanCreateTableBucket(principal, principal, "") {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
|
||||
return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
|
||||
}
|
||||
|
||||
var req CreateTableBucketRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate bucket name
|
||||
if err := validateBucketName(req.Name); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketPath := getTableBucketPath(req.Name)
|
||||
|
||||
// Check if bucket already exists and ensure no conflict with object store buckets
|
||||
tableBucketExists := false
|
||||
s3BucketExists := false
|
||||
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(r.Context(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucketsPath := resp.DirBuckets
|
||||
if bucketsPath == "" {
|
||||
bucketsPath = s3_constants.DefaultBucketsPath
|
||||
}
|
||||
_, err = filer_pb.LookupEntry(r.Context(), client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: bucketsPath,
|
||||
Name: req.Name,
|
||||
})
|
||||
if err != nil {
|
||||
if !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
s3BucketExists = true
|
||||
}
|
||||
_, err = filer_pb.LookupEntry(r.Context(), client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: TablesPath,
|
||||
Name: req.Name,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
tableBucketExists = true
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("S3Tables: failed to check bucket existence: %v", err)
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to check bucket existence")
|
||||
return err
|
||||
}
|
||||
|
||||
if s3BucketExists {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeBucketAlreadyExists, fmt.Sprintf("bucket name %s is already used by an object store bucket", req.Name))
|
||||
return fmt.Errorf("bucket name conflicts with object store bucket")
|
||||
}
|
||||
|
||||
if tableBucketExists {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeBucketAlreadyExists, fmt.Sprintf("table bucket %s already exists", req.Name))
|
||||
return fmt.Errorf("bucket already exists")
|
||||
}
|
||||
|
||||
// Create the bucket directory and set metadata as extended attributes
|
||||
now := time.Now()
|
||||
metadata := &tableBucketMetadata{
|
||||
Name: req.Name,
|
||||
CreatedAt: now,
|
||||
OwnerAccountID: h.getAccountID(r),
|
||||
}
|
||||
|
||||
metadataBytes, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
glog.Errorf("S3Tables: failed to marshal metadata: %v", err)
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal metadata")
|
||||
return err
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Ensure root tables directory exists
|
||||
if !h.entryExists(r.Context(), client, TablesPath) {
|
||||
if err := h.createDirectory(r.Context(), client, TablesPath); err != nil {
|
||||
return fmt.Errorf("failed to create root tables directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create bucket directory
|
||||
if err := h.createDirectory(r.Context(), client, bucketPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set metadata as extended attribute
|
||||
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata, metadataBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set tags if provided
|
||||
if len(req.Tags) > 0 {
|
||||
tagsBytes, err := json.Marshal(req.Tags)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal tags: %w", err)
|
||||
}
|
||||
if err := h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyTags, tagsBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("S3Tables: failed to create table bucket %s: %v", req.Name, err)
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create table bucket")
|
||||
return err
|
||||
}
|
||||
|
||||
resp := &CreateTableBucketResponse{
|
||||
ARN: h.generateTableBucketARN(metadata.OwnerAccountID, req.Name),
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// handleGetTableBucket gets details of a table bucket
|
||||
func (h *S3TablesHandler) handleGetTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
// Check permission
|
||||
|
||||
var req GetTableBucketRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
|
||||
var metadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table bucket: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("GetTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket details")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
resp := &GetTableBucketResponse{
|
||||
ARN: h.generateTableBucketARN(metadata.OwnerAccountID, bucketName),
|
||||
Name: metadata.Name,
|
||||
OwnerAccountID: metadata.OwnerAccountID,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleListTableBuckets lists all table buckets
|
||||
func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req ListTableBucketsRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
principal := h.getAccountID(r)
|
||||
accountID := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("ListTableBuckets", principal, accountID, "", "", &PolicyContext{
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to list table buckets")
|
||||
return NewAuthError("ListTableBuckets", principal, "not authorized to list table buckets")
|
||||
}
|
||||
|
||||
maxBuckets := req.MaxBuckets
|
||||
if maxBuckets <= 0 {
|
||||
maxBuckets = 100
|
||||
}
|
||||
// Cap to prevent uint32 overflow when used in uint32(maxBuckets*2)
|
||||
const maxBucketsLimit = 1000
|
||||
if maxBuckets > maxBucketsLimit {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "MaxBuckets exceeds maximum allowed value")
|
||||
return fmt.Errorf("invalid maxBuckets value: %d", maxBuckets)
|
||||
}
|
||||
|
||||
var buckets []TableBucketSummary
|
||||
|
||||
lastFileName := req.ContinuationToken
|
||||
err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
for len(buckets) < maxBuckets {
|
||||
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
|
||||
Directory: TablesPath,
|
||||
Limit: uint32(maxBuckets * 2), // Fetch more than needed to account for filtering
|
||||
StartFromFileName: lastFileName,
|
||||
InclusiveStartFrom: lastFileName == "" || lastFileName == req.ContinuationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hasMore := false
|
||||
for {
|
||||
entry, respErr := resp.Recv()
|
||||
if respErr != nil {
|
||||
if respErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return respErr
|
||||
}
|
||||
if entry.Entry == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip the start item if it was included in the previous page
|
||||
if len(buckets) == 0 && req.ContinuationToken != "" && entry.Entry.Name == req.ContinuationToken {
|
||||
continue
|
||||
}
|
||||
|
||||
hasMore = true
|
||||
lastFileName = entry.Entry.Name
|
||||
|
||||
if !entry.Entry.IsDirectory {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip entries starting with "."
|
||||
if strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Apply prefix filter
|
||||
if req.Prefix != "" && !strings.HasPrefix(entry.Entry.Name, req.Prefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Read metadata from extended attribute
|
||||
data, ok := entry.Entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
var metadata tableBucketMetadata
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
bucketPath := getTableBucketPath(entry.Entry.Name)
|
||||
bucketPolicy := ""
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, entry.Entry.Name)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("GetTableBucket", accountID, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: entry.Entry.Name,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
|
||||
buckets = append(buckets, TableBucketSummary{
|
||||
ARN: bucketARN,
|
||||
Name: entry.Entry.Name,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
})
|
||||
|
||||
if len(buckets) >= maxBuckets {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if !hasMore {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
// Check if it's a "not found" error - return empty list in that case
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
buckets = []TableBucketSummary{}
|
||||
} else {
|
||||
// For other errors, return error response
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list table buckets: %v", err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
paginationToken := ""
|
||||
if len(buckets) >= maxBuckets {
|
||||
paginationToken = lastFileName
|
||||
}
|
||||
|
||||
resp := &ListTableBucketsResponse{
|
||||
TableBuckets: buckets,
|
||||
ContinuationToken: paginationToken,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDeleteTableBucket deletes a table bucket
|
||||
func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req DeleteTableBucketRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
|
||||
// Check if bucket exists and perform ownership + emptiness check in one block
|
||||
var metadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
hasChildren := false
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// 1. Get metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
// No bucket policy set; proceed with empty bucketPolicy
|
||||
} else {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %w", err)
|
||||
}
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("DeleteTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
return NewAuthError("DeleteTableBucket", principal, fmt.Sprintf("not authorized to delete bucket %s", bucketName))
|
||||
}
|
||||
|
||||
// 3. Check if bucket is empty
|
||||
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
|
||||
Directory: bucketPath,
|
||||
Limit: 10,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
entry, err := resp.Recv()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
if entry.Entry != nil && !strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
hasChildren = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else if isAuthError(err) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to delete table bucket: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if hasChildren {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeBucketNotEmpty, "table bucket is not empty")
|
||||
return fmt.Errorf("bucket not empty")
|
||||
}
|
||||
|
||||
// Delete the bucket
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.deleteDirectory(r.Context(), client, bucketPath)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,556 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// handleCreateNamespace creates a new namespace in a table bucket
|
||||
func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req CreateNamespaceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
if len(req.Namespace) == 0 {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "namespace is required")
|
||||
return fmt.Errorf("namespace is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if table bucket exists
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
var bucketMetadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("CreateNamespace", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create namespace in this bucket")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
namespacePath := getNamespacePath(bucketName, namespaceName)
|
||||
|
||||
// Check if namespace already exists
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
_, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
|
||||
return err
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeNamespaceAlreadyExists, fmt.Sprintf("namespace %s already exists", namespaceName))
|
||||
return fmt.Errorf("namespace already exists")
|
||||
} else if !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check namespace: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// Create the namespace with bucket owner to maintain consistency
|
||||
// (authorization above ensures the caller has permission to create in this bucket)
|
||||
now := time.Now()
|
||||
metadata := &namespaceMetadata{
|
||||
Namespace: req.Namespace,
|
||||
CreatedAt: now,
|
||||
OwnerAccountID: bucketMetadata.OwnerAccountID,
|
||||
}
|
||||
|
||||
metadataBytes, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal namespace metadata")
|
||||
return fmt.Errorf("failed to marshal metadata: %w", err)
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Create namespace directory
|
||||
if err := h.createDirectory(r.Context(), client, namespacePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set metadata as extended attribute
|
||||
if err := h.setExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata, metadataBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create namespace")
|
||||
return err
|
||||
}
|
||||
|
||||
resp := &CreateNamespaceResponse{
|
||||
Namespace: req.Namespace,
|
||||
TableBucketARN: req.TableBucketARN,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetNamespace gets details of a namespace
|
||||
func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req GetNamespaceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
namespacePath := getNamespacePath(bucketName, namespaceName)
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
|
||||
// Get namespace and bucket policy
|
||||
var metadata namespaceMetadata
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get namespace: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("GetNamespace", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
resp := &GetNamespaceResponse{
|
||||
Namespace: metadata.Namespace,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
OwnerAccountID: metadata.OwnerAccountID,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleListNamespaces lists all namespaces in a table bucket
|
||||
func (h *S3TablesHandler) handleListNamespaces(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req ListNamespacesRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
maxNamespaces := req.MaxNamespaces
|
||||
if maxNamespaces <= 0 {
|
||||
maxNamespaces = 100
|
||||
}
|
||||
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
|
||||
// Check permission (check bucket ownership)
|
||||
var bucketMetadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list namespaces: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("ListNamespaces", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
var namespaces []NamespaceSummary
|
||||
|
||||
lastFileName := req.ContinuationToken
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
for len(namespaces) < maxNamespaces {
|
||||
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
|
||||
Directory: bucketPath,
|
||||
Limit: uint32(maxNamespaces * 2),
|
||||
StartFromFileName: lastFileName,
|
||||
InclusiveStartFrom: lastFileName == "" || lastFileName == req.ContinuationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hasMore := false
|
||||
for {
|
||||
entry, respErr := resp.Recv()
|
||||
if respErr != nil {
|
||||
if respErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return respErr
|
||||
}
|
||||
if entry.Entry == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip the start item if it was included in the previous page
|
||||
if len(namespaces) == 0 && req.ContinuationToken != "" && entry.Entry.Name == req.ContinuationToken {
|
||||
continue
|
||||
}
|
||||
|
||||
hasMore = true
|
||||
lastFileName = entry.Entry.Name
|
||||
|
||||
if !entry.Entry.IsDirectory {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip hidden entries
|
||||
if strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Apply prefix filter
|
||||
if req.Prefix != "" && !strings.HasPrefix(entry.Entry.Name, req.Prefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Read metadata from extended attribute
|
||||
data, ok := entry.Entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
var metadata namespaceMetadata
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if metadata.OwnerAccountID != bucketMetadata.OwnerAccountID {
|
||||
continue
|
||||
}
|
||||
|
||||
namespaces = append(namespaces, NamespaceSummary{
|
||||
Namespace: metadata.Namespace,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
})
|
||||
|
||||
if len(namespaces) >= maxNamespaces {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if !hasMore {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
namespaces = []NamespaceSummary{}
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list namespaces: %v", err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
paginationToken := ""
|
||||
if len(namespaces) >= maxNamespaces {
|
||||
paginationToken = lastFileName
|
||||
}
|
||||
|
||||
resp := &ListNamespacesResponse{
|
||||
Namespaces: namespaces,
|
||||
ContinuationToken: paginationToken,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDeleteNamespace deletes a namespace from a table bucket
|
||||
func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req DeleteNamespaceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
namespacePath := getNamespacePath(bucketName, namespaceName)
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
|
||||
// Check if namespace exists and get metadata for permission check
|
||||
var metadata namespaceMetadata
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get namespace metadata: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(metadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("DeleteNamespace", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// Check if namespace is empty
|
||||
hasChildren := false
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.ListEntries(r.Context(), &filer_pb.ListEntriesRequest{
|
||||
Directory: namespacePath,
|
||||
Limit: 10,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
entry, err := resp.Recv()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return err
|
||||
}
|
||||
if entry.Entry != nil && !strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
hasChildren = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", flattenNamespace(req.Namespace)))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list namespace entries: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if hasChildren {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeNamespaceNotEmpty, "namespace is not empty")
|
||||
return fmt.Errorf("namespace not empty")
|
||||
}
|
||||
|
||||
// Delete the namespace
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.deleteDirectory(r.Context(), client, namespacePath)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete namespace")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,927 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// extractResourceOwnerAndBucket extracts ownership info and bucket name from resource metadata.
|
||||
// This helper consolidates the repeated pattern used in handleTagResource, handleListTagsForResource,
|
||||
// and handleUntagResource.
|
||||
func (h *S3TablesHandler) extractResourceOwnerAndBucket(
|
||||
data []byte,
|
||||
resourcePath string,
|
||||
rType ResourceType,
|
||||
) (ownerAccountID, bucketName string, err error) {
|
||||
// Extract bucket name from resource path (format: /table-buckets/{bucket}/... for both tables and buckets)
|
||||
parts := strings.Split(strings.Trim(resourcePath, "/"), "/")
|
||||
if len(parts) >= 2 {
|
||||
bucketName = parts[1]
|
||||
}
|
||||
|
||||
if rType == ResourceTypeTable {
|
||||
var meta tableMetadataInternal
|
||||
if err := json.Unmarshal(data, &meta); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
ownerAccountID = meta.OwnerAccountID
|
||||
} else {
|
||||
var meta tableBucketMetadata
|
||||
if err := json.Unmarshal(data, &meta); err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
ownerAccountID = meta.OwnerAccountID
|
||||
}
|
||||
return ownerAccountID, bucketName, nil
|
||||
}
|
||||
|
||||
// handlePutTableBucketPolicy puts a policy on a table bucket
|
||||
func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req PutTableBucketPolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
if req.ResourcePolicy == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourcePolicy is required")
|
||||
return fmt.Errorf("resourcePolicy is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if bucket exists and get metadata for ownership check
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
var bucketMetadata tableBucketMetadata
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("PutTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, "", bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table bucket policy")
|
||||
return NewAuthError("PutTableBucketPolicy", principal, "not authorized to put table bucket policy")
|
||||
}
|
||||
|
||||
// Write policy
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.setExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy, []byte(req.ResourcePolicy))
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to put table bucket policy")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetTableBucketPolicy gets the policy of a table bucket
|
||||
func (h *S3TablesHandler) handleGetTableBucketPolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req GetTableBucketPolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
var policy []byte
|
||||
var bucketMetadata tableBucketMetadata
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Get metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
|
||||
// Get policy
|
||||
policy, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
return err
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchPolicy, "table bucket policy not found")
|
||||
return err
|
||||
}
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table bucket policy: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("GetTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, string(policy), bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket policy")
|
||||
return NewAuthError("GetTableBucketPolicy", principal, "not authorized to get table bucket policy")
|
||||
}
|
||||
|
||||
resp := &GetTableBucketPolicyResponse{
|
||||
ResourcePolicy: string(policy),
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDeleteTableBucketPolicy deletes the policy of a table bucket
|
||||
func (h *S3TablesHandler) handleDeleteTableBucketPolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req DeleteTableBucketPolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
|
||||
// Check if bucket exists and get metadata for ownership check
|
||||
var bucketMetadata tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table bucket: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("DeleteTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table bucket policy")
|
||||
return NewAuthError("DeleteTableBucketPolicy", principal, "not authorized to delete table bucket policy")
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.deleteExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
})
|
||||
|
||||
if err != nil && !errors.Is(err, ErrAttributeNotFound) {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table bucket policy")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handlePutTablePolicy puts a policy on a table
|
||||
func (h *S3TablesHandler) handlePutTablePolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req PutTablePolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
|
||||
return fmt.Errorf("missing required parameters")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.ResourcePolicy == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourcePolicy is required")
|
||||
return fmt.Errorf("resourcePolicy is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if table exists
|
||||
tableName, err := validateTableName(req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
tablePath := getTablePath(bucketName, namespaceName, tableName)
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
|
||||
var metadata tableMetadataInternal
|
||||
var bucketPolicy string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("PutTablePolicy", principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table policy")
|
||||
return NewAuthError("PutTablePolicy", principal, "not authorized to put table policy")
|
||||
}
|
||||
|
||||
// Write policy
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.setExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy, []byte(req.ResourcePolicy))
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to put table policy")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetTablePolicy gets the policy of a table
|
||||
func (h *S3TablesHandler) handleGetTablePolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req GetTablePolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
|
||||
return fmt.Errorf("missing required parameters")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
tableName, err := validateTableName(req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
tablePath := getTablePath(bucketName, namespaceName, tableName)
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
var policy []byte
|
||||
var metadata tableMetadataInternal
|
||||
var bucketPolicy string
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Get metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
|
||||
}
|
||||
|
||||
// Get policy
|
||||
policy, err = h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
return err
|
||||
}
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchPolicy, "table policy not found")
|
||||
return err
|
||||
}
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table policy: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("GetTablePolicy", principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table policy")
|
||||
return NewAuthError("GetTablePolicy", principal, "not authorized to get table policy")
|
||||
}
|
||||
|
||||
resp := &GetTablePolicyResponse{
|
||||
ResourcePolicy: string(policy),
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleDeleteTablePolicy deletes the policy of a table
|
||||
func (h *S3TablesHandler) handleDeleteTablePolicy(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req DeleteTablePolicyRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
|
||||
return fmt.Errorf("missing required parameters")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
tableName, err := validateTableName(req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
tablePath := getTablePath(bucketName, namespaceName, tableName)
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
|
||||
// Check if table exists
|
||||
var metadata tableMetadataInternal
|
||||
var bucketPolicy string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("DeleteTablePolicy", principal, metadata.OwnerAccountID, bucketPolicy, tableARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table policy")
|
||||
return NewAuthError("DeleteTablePolicy", principal, "not authorized to delete table policy")
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.deleteExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy)
|
||||
})
|
||||
|
||||
if err != nil && !errors.Is(err, ErrAttributeNotFound) {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table policy")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleTagResource adds tags to a resource
|
||||
func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req TagResourceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.ResourceARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourceArn is required")
|
||||
return fmt.Errorf("resourceArn is required")
|
||||
}
|
||||
|
||||
if len(req.Tags) == 0 {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tags are required")
|
||||
return fmt.Errorf("tags are required")
|
||||
}
|
||||
|
||||
// Parse resource ARN to determine if it's a bucket or table
|
||||
resourcePath, extendedKey, rType, err := h.resolveResourcePath(req.ResourceARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Read existing tags and merge, AND check permissions based on metadata ownership
|
||||
existingTags := make(map[string]string)
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
requestTagKeys := mapKeys(req.Tags)
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Read metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if we have a bucket name
|
||||
if bucketName != "" {
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Read existing tags
|
||||
data, err = h.getExtendedAttribute(r.Context(), client, resourcePath, extendedKey)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return err
|
||||
}
|
||||
} else if err := json.Unmarshal(data, &existingTags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resourceARN := req.ResourceARN
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("TagResource", principal, ownerAccountID, bucketPolicy, resourceARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
RequestTags: req.Tags,
|
||||
TagKeys: requestTagKeys,
|
||||
ResourceTags: existingTags,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
return NewAuthError("TagResource", principal, "not authorized to tag resource")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
errorCode := ErrCodeNoSuchBucket
|
||||
if rType == ResourceTypeTable {
|
||||
errorCode = ErrCodeNoSuchTable
|
||||
}
|
||||
h.writeError(w, http.StatusNotFound, errorCode, "resource not found")
|
||||
} else if isAuthError(err) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to read existing tags: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Merge new tags
|
||||
for k, v := range req.Tags {
|
||||
existingTags[k] = v
|
||||
}
|
||||
|
||||
// Write merged tags
|
||||
tagsBytes, err := json.Marshal(existingTags)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags")
|
||||
return fmt.Errorf("failed to marshal tags: %w", err)
|
||||
}
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.setExtendedAttribute(r.Context(), client, resourcePath, extendedKey, tagsBytes)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to tag resource")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleListTagsForResource lists tags for a resource
|
||||
func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req ListTagsForResourceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.ResourceARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourceArn is required")
|
||||
return fmt.Errorf("resourceArn is required")
|
||||
}
|
||||
|
||||
resourcePath, extendedKey, rType, err := h.resolveResourcePath(req.ResourceARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
tags := make(map[string]string)
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Read metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if we have a bucket name
|
||||
if bucketName != "" {
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
data, err = h.getExtendedAttribute(r.Context(), client, resourcePath, extendedKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
return nil // No tags is not an error.
|
||||
}
|
||||
return err // Propagate other errors.
|
||||
}
|
||||
if err := json.Unmarshal(data, &tags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resourceARN := req.ResourceARN
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("ListTagsForResource", principal, ownerAccountID, bucketPolicy, resourceARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tags,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
return NewAuthError("ListTagsForResource", principal, "not authorized to list tags for resource")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
errorCode := ErrCodeNoSuchBucket
|
||||
if rType == ResourceTypeTable {
|
||||
errorCode = ErrCodeNoSuchTable
|
||||
}
|
||||
h.writeError(w, http.StatusNotFound, errorCode, "resource not found")
|
||||
} else if isAuthError(err) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list tags: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
resp := &ListTagsForResourceResponse{
|
||||
Tags: tags,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleUntagResource removes tags from a resource
|
||||
func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
var req UntagResourceRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.ResourceARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "resourceArn is required")
|
||||
return fmt.Errorf("resourceArn is required")
|
||||
}
|
||||
|
||||
if len(req.TagKeys) == 0 {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tagKeys are required")
|
||||
return fmt.Errorf("tagKeys are required")
|
||||
}
|
||||
|
||||
resourcePath, extendedKey, rType, err := h.resolveResourcePath(req.ResourceARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Read existing tags, check permission
|
||||
tags := make(map[string]string)
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Read metadata for ownership check
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, resourcePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ownerAccountID, bucketName, err := h.extractResourceOwnerAndBucket(data, resourcePath, rType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if we have a bucket name
|
||||
if bucketName != "" {
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to read bucket policy: %w", err)
|
||||
}
|
||||
// Policy not found is not an error; bucketPolicy remains empty
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
data, err = h.getExtendedAttribute(r.Context(), client, resourcePath, extendedKey)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &tags); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resourceARN := req.ResourceARN
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("UntagResource", principal, ownerAccountID, bucketPolicy, resourceARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
TagKeys: req.TagKeys,
|
||||
ResourceTags: tags,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
return NewAuthError("UntagResource", principal, "not authorized to untag resource")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
errorCode := ErrCodeNoSuchBucket
|
||||
if rType == ResourceTypeTable {
|
||||
errorCode = ErrCodeNoSuchTable
|
||||
}
|
||||
h.writeError(w, http.StatusNotFound, errorCode, "resource not found")
|
||||
} else if isAuthError(err) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, err.Error())
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to read existing tags")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove specified tags
|
||||
for _, key := range req.TagKeys {
|
||||
delete(tags, key)
|
||||
}
|
||||
|
||||
// Write updated tags
|
||||
tagsBytes, err := json.Marshal(tags)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal tags")
|
||||
return fmt.Errorf("failed to marshal tags: %w", err)
|
||||
}
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.setExtendedAttribute(r.Context(), client, resourcePath, extendedKey, tagsBytes)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to untag resource")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveResourcePath determines the resource path and extended attribute key from a resource ARN
|
||||
func (h *S3TablesHandler) resolveResourcePath(resourceARN string) (path string, key string, rType ResourceType, err error) {
|
||||
// Try parsing as table ARN first
|
||||
bucketName, namespace, tableName, err := parseTableFromARN(resourceARN)
|
||||
if err == nil {
|
||||
return getTablePath(bucketName, namespace, tableName), ExtendedKeyTags, ResourceTypeTable, nil
|
||||
}
|
||||
|
||||
// Try parsing as bucket ARN
|
||||
bucketName, err = parseBucketNameFromARN(resourceARN)
|
||||
if err == nil {
|
||||
return getTableBucketPath(bucketName), ExtendedKeyTags, ResourceTypeBucket, nil
|
||||
}
|
||||
|
||||
return "", "", "", fmt.Errorf("invalid resource ARN: %s", resourceARN)
|
||||
}
|
||||
@@ -0,0 +1,918 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// handleCreateTable creates a new table in a namespace
|
||||
func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req CreateTableRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.Name == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "name is required")
|
||||
return fmt.Errorf("name is required")
|
||||
}
|
||||
|
||||
if req.Format == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "format is required")
|
||||
return fmt.Errorf("format is required")
|
||||
}
|
||||
|
||||
// Validate format
|
||||
if req.Format != "ICEBERG" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "only ICEBERG format is supported")
|
||||
return fmt.Errorf("invalid format")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate table name
|
||||
tableName, err := validateTableName(req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if namespace exists
|
||||
namespacePath := getNamespacePath(bucketName, namespaceName)
|
||||
var namespaceMetadata namespaceMetadata
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &namespaceMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal namespace metadata: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, fmt.Sprintf("namespace %s not found", namespaceName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check namespace: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Authorize table creation using policy framework (namespace + bucket policies)
|
||||
accountID := h.getAccountID(r)
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
namespacePolicy := ""
|
||||
bucketPolicy := ""
|
||||
bucketTags := map[string]string{}
|
||||
var data []byte
|
||||
var bucketMetadata tableBucketMetadata
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Fetch bucket metadata to use correct owner for bucket policy evaluation
|
||||
data, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err == nil {
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket metadata: %v", err)
|
||||
}
|
||||
|
||||
// Fetch namespace policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
namespacePolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch namespace policy: %v", err)
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
if tags, err := h.readTags(r.Context(), client, bucketPath); err != nil {
|
||||
return err
|
||||
} else if tags != nil {
|
||||
bucketTags = tags
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to fetch policies: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
identityActions := getIdentityActions(r)
|
||||
nsAllowed := CheckPermissionWithContext("CreateTable", accountID, namespaceMetadata.OwnerAccountID, namespacePolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
RequestTags: req.Tags,
|
||||
TagKeys: mapKeys(req.Tags),
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
})
|
||||
bucketAllowed := CheckPermissionWithContext("CreateTable", accountID, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
RequestTags: req.Tags,
|
||||
TagKeys: mapKeys(req.Tags),
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
})
|
||||
|
||||
if !nsAllowed && !bucketAllowed {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table in this namespace")
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
tablePath := getTablePath(bucketName, namespaceName, tableName)
|
||||
|
||||
// Check if table already exists
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
_, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
|
||||
return err
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeTableAlreadyExists, fmt.Sprintf("table %s already exists", tableName))
|
||||
return fmt.Errorf("table already exists")
|
||||
} else if !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// Create the table
|
||||
now := time.Now()
|
||||
versionToken := generateVersionToken()
|
||||
|
||||
metadata := &tableMetadataInternal{
|
||||
Name: tableName,
|
||||
Namespace: namespaceName,
|
||||
Format: req.Format,
|
||||
CreatedAt: now,
|
||||
ModifiedAt: now,
|
||||
OwnerAccountID: namespaceMetadata.OwnerAccountID, // Inherit namespace owner for consistency
|
||||
VersionToken: versionToken,
|
||||
Metadata: req.Metadata,
|
||||
}
|
||||
|
||||
metadataBytes, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to marshal table metadata")
|
||||
return fmt.Errorf("failed to marshal metadata: %w", err)
|
||||
}
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Create table directory
|
||||
if err := h.createDirectory(r.Context(), client, tablePath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create data subdirectory for Iceberg files
|
||||
dataPath := tablePath + "/data"
|
||||
if err := h.createDirectory(r.Context(), client, dataPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set metadata as extended attribute
|
||||
if err := h.setExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata, metadataBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Set tags if provided
|
||||
if len(req.Tags) > 0 {
|
||||
tagsBytes, err := json.Marshal(req.Tags)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal tags: %w", err)
|
||||
}
|
||||
if err := h.setExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyTags, tagsBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to create table")
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
|
||||
|
||||
resp := &CreateTableResponse{
|
||||
TableARN: tableARN,
|
||||
VersionToken: versionToken,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleGetTable gets details of a table
|
||||
func (h *S3TablesHandler) handleGetTable(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req GetTableRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
var bucketName, namespace, tableName string
|
||||
var err error
|
||||
|
||||
// Support getting by ARN or by bucket/namespace/name
|
||||
if req.TableARN != "" {
|
||||
bucketName, namespace, tableName, err = parseTableFromARN(req.TableARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
} else if req.TableBucketARN != "" && len(req.Namespace) > 0 && req.Name != "" {
|
||||
bucketName, err = parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
namespace, err = validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
tableName, err = validateTableName(req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "either tableARN or (tableBucketARN, namespace, name) is required")
|
||||
return fmt.Errorf("missing required parameters")
|
||||
}
|
||||
|
||||
tablePath := getTablePath(bucketName, namespace, tableName)
|
||||
|
||||
var metadata tableMetadataInternal
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to get table: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Authorize access to the table using policy framework
|
||||
accountID := h.getAccountID(r)
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
tablePolicy := ""
|
||||
bucketPolicy := ""
|
||||
bucketTags := map[string]string{}
|
||||
tableTags := map[string]string{}
|
||||
var bucketMetadata tableBucketMetadata
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Fetch bucket metadata to use correct owner for bucket policy evaluation
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err == nil {
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket metadata: %v", err)
|
||||
}
|
||||
|
||||
// Fetch table policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
tablePolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch table policy: %v", err)
|
||||
}
|
||||
if tags, err := h.readTags(r.Context(), client, tablePath); err != nil {
|
||||
return err
|
||||
} else if tags != nil {
|
||||
tableTags = tags
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
if tags, err := h.readTags(r.Context(), client, bucketPath); err != nil {
|
||||
return err
|
||||
} else if tags != nil {
|
||||
bucketTags = tags
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to fetch policies: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespace+"/"+tableName)
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
identityActions := getIdentityActions(r)
|
||||
tableAllowed := CheckPermissionWithContext("GetTable", accountID, metadata.OwnerAccountID, tablePolicy, tableARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespace,
|
||||
TableName: tableName,
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tableTags,
|
||||
IdentityActions: identityActions,
|
||||
})
|
||||
bucketAllowed := CheckPermissionWithContext("GetTable", accountID, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespace,
|
||||
TableName: tableName,
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tableTags,
|
||||
IdentityActions: identityActions,
|
||||
})
|
||||
|
||||
if !tableAllowed && !bucketAllowed {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
resp := &GetTableResponse{
|
||||
Name: metadata.Name,
|
||||
TableARN: tableARN,
|
||||
Namespace: []string{metadata.Namespace},
|
||||
Format: metadata.Format,
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
ModifiedAt: metadata.ModifiedAt,
|
||||
OwnerAccountID: metadata.OwnerAccountID,
|
||||
MetadataLocation: metadata.MetadataLocation,
|
||||
VersionToken: metadata.VersionToken,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleListTables lists all tables in a namespace or bucket
|
||||
func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req ListTablesRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN is required")
|
||||
return fmt.Errorf("tableBucketARN is required")
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
maxTables := req.MaxTables
|
||||
if maxTables <= 0 {
|
||||
maxTables = 100
|
||||
}
|
||||
// Cap to prevent uint32 overflow when used in uint32(maxTables*2)
|
||||
const maxTablesLimit = 1000
|
||||
if maxTables > maxTablesLimit {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "MaxTables exceeds maximum allowed value")
|
||||
return fmt.Errorf("invalid maxTables value: %d", maxTables)
|
||||
}
|
||||
|
||||
// Pre-validate namespace before calling WithFilerClient to return 400 on validation errors
|
||||
var namespaceName string
|
||||
if len(req.Namespace) > 0 {
|
||||
var err error
|
||||
namespaceName, err = validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var tables []TableSummary
|
||||
var paginationToken string
|
||||
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
var err error
|
||||
accountID := h.getAccountID(r)
|
||||
|
||||
if len(req.Namespace) > 0 {
|
||||
// Namespace has already been validated above
|
||||
namespacePath := getNamespacePath(bucketName, namespaceName)
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
var nsMeta namespaceMetadata
|
||||
var bucketMeta tableBucketMetadata
|
||||
var namespacePolicy, bucketPolicy string
|
||||
bucketTags := map[string]string{}
|
||||
|
||||
// Fetch namespace metadata and policy
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err // Not Found handled by caller
|
||||
}
|
||||
if err := json.Unmarshal(data, &nsMeta); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch namespace policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, namespacePath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
namespacePolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch namespace policy: %v", err)
|
||||
}
|
||||
|
||||
// Fetch bucket metadata and policy
|
||||
data, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err == nil {
|
||||
if err := json.Unmarshal(data, &bucketMeta); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket metadata: %v", err)
|
||||
}
|
||||
|
||||
policyData, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
if tags, err := h.readTags(r.Context(), client, bucketPath); err != nil {
|
||||
return err
|
||||
} else if tags != nil {
|
||||
bucketTags = tags
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMeta.OwnerAccountID, bucketName)
|
||||
identityActions := getIdentityActions(r)
|
||||
nsAllowed := CheckPermissionWithContext("ListTables", accountID, nsMeta.OwnerAccountID, namespacePolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
})
|
||||
bucketAllowed := CheckPermissionWithContext("ListTables", accountID, bucketMeta.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
})
|
||||
if !nsAllowed && !bucketAllowed {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
tables, paginationToken, err = h.listTablesInNamespaceWithClient(r, client, bucketName, namespaceName, req.Prefix, req.ContinuationToken, maxTables)
|
||||
} else {
|
||||
// List tables across all namespaces in bucket
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
var bucketMeta tableBucketMetadata
|
||||
var bucketPolicy string
|
||||
bucketTags := map[string]string{}
|
||||
|
||||
// Fetch bucket metadata and policy
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(data, &bucketMeta); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fetch bucket policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err == nil {
|
||||
bucketPolicy = string(policyData)
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %v", err)
|
||||
}
|
||||
if tags, err := h.readTags(r.Context(), client, bucketPath); err != nil {
|
||||
return err
|
||||
} else if tags != nil {
|
||||
bucketTags = tags
|
||||
}
|
||||
|
||||
bucketARN := h.generateTableBucketARN(bucketMeta.OwnerAccountID, bucketName)
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("ListTables", accountID, bucketMeta.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
}) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
tables, paginationToken, err = h.listTablesInAllNamespaces(r, client, bucketName, req.Prefix, req.ContinuationToken, maxTables)
|
||||
}
|
||||
return err
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
// If the bucket or namespace directory is not found, return an empty result
|
||||
tables = []TableSummary{}
|
||||
paginationToken = ""
|
||||
} else if isAuthError(err) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "Access Denied")
|
||||
return err
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to list tables: %v", err))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
resp := &ListTablesResponse{
|
||||
Tables: tables,
|
||||
ContinuationToken: paginationToken,
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// listTablesInNamespaceWithClient lists tables in a specific namespace
|
||||
func (h *S3TablesHandler) listTablesInNamespaceWithClient(r *http.Request, client filer_pb.SeaweedFilerClient, bucketName, namespaceName, prefix, continuationToken string, maxTables int) ([]TableSummary, string, error) {
|
||||
namespacePath := getNamespacePath(bucketName, namespaceName)
|
||||
return h.listTablesWithClient(r, client, namespacePath, bucketName, namespaceName, prefix, continuationToken, maxTables)
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) listTablesWithClient(r *http.Request, client filer_pb.SeaweedFilerClient, dirPath, bucketName, namespaceName, prefix, continuationToken string, maxTables int) ([]TableSummary, string, error) {
|
||||
var tables []TableSummary
|
||||
lastFileName := continuationToken
|
||||
ctx := r.Context()
|
||||
|
||||
for len(tables) < maxTables {
|
||||
resp, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
|
||||
Directory: dirPath,
|
||||
Limit: uint32(maxTables * 2),
|
||||
StartFromFileName: lastFileName,
|
||||
InclusiveStartFrom: lastFileName == "" || lastFileName == continuationToken,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
hasMore := false
|
||||
for {
|
||||
entry, respErr := resp.Recv()
|
||||
if respErr != nil {
|
||||
if respErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return nil, "", respErr
|
||||
}
|
||||
if entry.Entry == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip the start item if it was included in the previous page
|
||||
if len(tables) == 0 && continuationToken != "" && entry.Entry.Name == continuationToken {
|
||||
continue
|
||||
}
|
||||
|
||||
hasMore = true
|
||||
lastFileName = entry.Entry.Name
|
||||
|
||||
if !entry.Entry.IsDirectory {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip hidden entries
|
||||
if strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Apply prefix filter
|
||||
if prefix != "" && !strings.HasPrefix(entry.Entry.Name, prefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Read table metadata from extended attribute
|
||||
data, ok := entry.Entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
var metadata tableMetadataInternal
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Note: Authorization (ownership or policy-based access) is checked at the handler level
|
||||
// before calling this function. This filter is removed to allow policy-based sharing.
|
||||
// The caller has already been verified to have ListTables permission for this namespace/bucket.
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+entry.Entry.Name)
|
||||
|
||||
tables = append(tables, TableSummary{
|
||||
Name: entry.Entry.Name,
|
||||
TableARN: tableARN,
|
||||
Namespace: []string{namespaceName},
|
||||
CreatedAt: metadata.CreatedAt,
|
||||
ModifiedAt: metadata.ModifiedAt,
|
||||
})
|
||||
|
||||
if len(tables) >= maxTables {
|
||||
return tables, lastFileName, nil
|
||||
}
|
||||
}
|
||||
|
||||
if !hasMore {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(tables) < maxTables {
|
||||
lastFileName = ""
|
||||
}
|
||||
return tables, lastFileName, nil
|
||||
}
|
||||
|
||||
func (h *S3TablesHandler) listTablesInAllNamespaces(r *http.Request, client filer_pb.SeaweedFilerClient, bucketName, prefix, continuationToken string, maxTables int) ([]TableSummary, string, error) {
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
ctx := r.Context()
|
||||
|
||||
var continuationNamespace string
|
||||
var startTableName string
|
||||
if continuationToken != "" {
|
||||
if parts := strings.SplitN(continuationToken, "/", 2); len(parts) == 2 {
|
||||
continuationNamespace = parts[0]
|
||||
startTableName = parts[1]
|
||||
} else {
|
||||
continuationNamespace = continuationToken
|
||||
}
|
||||
}
|
||||
|
||||
var tables []TableSummary
|
||||
lastNamespace := continuationNamespace
|
||||
for {
|
||||
// List namespaces in batches
|
||||
resp, err := client.ListEntries(ctx, &filer_pb.ListEntriesRequest{
|
||||
Directory: bucketPath,
|
||||
Limit: 100,
|
||||
StartFromFileName: lastNamespace,
|
||||
InclusiveStartFrom: (lastNamespace == continuationNamespace && startTableName != "") || (lastNamespace == "" && continuationNamespace == ""),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
hasMore := false
|
||||
for {
|
||||
entry, respErr := resp.Recv()
|
||||
if respErr != nil {
|
||||
if respErr == io.EOF {
|
||||
break
|
||||
}
|
||||
return nil, "", respErr
|
||||
}
|
||||
if entry.Entry == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
hasMore = true
|
||||
lastNamespace = entry.Entry.Name
|
||||
|
||||
if !entry.Entry.IsDirectory || strings.HasPrefix(entry.Entry.Name, ".") {
|
||||
continue
|
||||
}
|
||||
|
||||
namespace := entry.Entry.Name
|
||||
tableNameFilter := ""
|
||||
if namespace == continuationNamespace {
|
||||
tableNameFilter = startTableName
|
||||
}
|
||||
|
||||
nsTables, nsToken, err := h.listTablesInNamespaceWithClient(r, client, bucketName, namespace, prefix, tableNameFilter, maxTables-len(tables))
|
||||
if err != nil {
|
||||
glog.Warningf("S3Tables: failed to list tables in namespace %s/%s: %v", bucketName, namespace, err)
|
||||
continue
|
||||
}
|
||||
|
||||
tables = append(tables, nsTables...)
|
||||
|
||||
if namespace == continuationNamespace {
|
||||
startTableName = ""
|
||||
}
|
||||
|
||||
if len(tables) >= maxTables {
|
||||
paginationToken := namespace + "/" + nsToken
|
||||
if nsToken == "" {
|
||||
// If we hit the limit exactly at the end of a namespace, the next token should be the next namespace
|
||||
paginationToken = namespace // This will start from the NEXT namespace in the outer loop
|
||||
}
|
||||
return tables, paginationToken, nil
|
||||
}
|
||||
}
|
||||
|
||||
if !hasMore {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return tables, "", nil
|
||||
}
|
||||
|
||||
// handleDeleteTable deletes a table from a namespace
|
||||
func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
|
||||
var req DeleteTableRequest
|
||||
if err := h.readRequestBody(r, &req); err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if req.TableBucketARN == "" || len(req.Namespace) == 0 || req.Name == "" {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, "tableBucketARN, namespace, and name are required")
|
||||
return fmt.Errorf("missing required parameters")
|
||||
}
|
||||
|
||||
namespaceName, err := validateNamespace(req.Namespace)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
bucketName, err := parseBucketNameFromARN(req.TableBucketARN)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
tableName, err := validateTableName(req.Name)
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusBadRequest, ErrCodeInvalidRequest, err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
tablePath := getTablePath(bucketName, namespaceName, tableName)
|
||||
|
||||
// Check if table exists and enforce VersionToken if provided
|
||||
var metadata tableMetadataInternal
|
||||
var tablePolicy string
|
||||
var bucketPolicy string
|
||||
var bucketTags map[string]string
|
||||
var tableTags map[string]string
|
||||
var bucketMetadata tableBucketMetadata
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
data, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyMetadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal table metadata: %w", err)
|
||||
}
|
||||
|
||||
if req.VersionToken != "" {
|
||||
if metadata.VersionToken != req.VersionToken {
|
||||
return ErrVersionTokenMismatch
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch table policy if it exists
|
||||
policyData, err := h.getExtendedAttribute(r.Context(), client, tablePath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrAttributeNotFound) {
|
||||
// No table policy set; proceed with empty policy
|
||||
} else {
|
||||
return fmt.Errorf("failed to fetch table policy: %w", err)
|
||||
}
|
||||
} else {
|
||||
tablePolicy = string(policyData)
|
||||
}
|
||||
|
||||
tableTags, err = h.readTags(r.Context(), client, tablePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bucketPath := getTableBucketPath(bucketName)
|
||||
data, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyMetadata)
|
||||
if err == nil {
|
||||
if err := json.Unmarshal(data, &bucketMetadata); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal bucket metadata: %w", err)
|
||||
}
|
||||
} else if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket metadata: %w", err)
|
||||
}
|
||||
policyData, err = h.getExtendedAttribute(r.Context(), client, bucketPath, ExtendedKeyPolicy)
|
||||
if err != nil {
|
||||
if !errors.Is(err, ErrAttributeNotFound) {
|
||||
return fmt.Errorf("failed to fetch bucket policy: %w", err)
|
||||
}
|
||||
} else {
|
||||
bucketPolicy = string(policyData)
|
||||
}
|
||||
bucketTags, err = h.readTags(r.Context(), client, bucketPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchTable, fmt.Sprintf("table %s not found", tableName))
|
||||
} else if errors.Is(err, ErrVersionTokenMismatch) {
|
||||
h.writeError(w, http.StatusConflict, ErrCodeConflict, "version token mismatch")
|
||||
} else {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, fmt.Sprintf("failed to check table: %v", err))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
tableARN := h.generateTableARN(metadata.OwnerAccountID, bucketName, namespaceName+"/"+tableName)
|
||||
bucketARN := h.generateTableBucketARN(bucketMetadata.OwnerAccountID, bucketName)
|
||||
principal := h.getAccountID(r)
|
||||
identityActions := getIdentityActions(r)
|
||||
tableAllowed := CheckPermissionWithContext("DeleteTable", principal, metadata.OwnerAccountID, tablePolicy, tableARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tableTags,
|
||||
IdentityActions: identityActions,
|
||||
})
|
||||
bucketAllowed := CheckPermissionWithContext("DeleteTable", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tableTags,
|
||||
IdentityActions: identityActions,
|
||||
})
|
||||
if !tableAllowed && !bucketAllowed {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table")
|
||||
return NewAuthError("DeleteTable", principal, "not authorized to delete table")
|
||||
}
|
||||
|
||||
// Delete the table
|
||||
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return h.deleteDirectory(r.Context(), client, tablePath)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
h.writeError(w, http.StatusInternalServerError, ErrCodeInternalError, "failed to delete table")
|
||||
return err
|
||||
}
|
||||
|
||||
h.writeJSON(w, http.StatusOK, nil)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,414 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
pathpkg "path"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// Iceberg file layout validation
|
||||
// Apache Iceberg tables follow a specific file layout structure:
|
||||
// - metadata/ directory containing metadata files (*.json, *.avro)
|
||||
// - data/ directory containing data files (*.parquet, *.orc, *.avro)
|
||||
//
|
||||
// Valid file patterns include:
|
||||
// - metadata/v*.metadata.json (table metadata)
|
||||
// - metadata/snap-*.avro (snapshot manifest lists)
|
||||
// - metadata/*.avro (manifest files)
|
||||
// - data/*.parquet, data/*.orc, data/*.avro (data files)
|
||||
|
||||
const uuidPattern = `[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}`
|
||||
|
||||
var (
|
||||
// Allowed directories in an Iceberg table
|
||||
icebergAllowedDirs = map[string]bool{
|
||||
"metadata": true,
|
||||
"data": true,
|
||||
}
|
||||
|
||||
// Patterns for valid metadata files
|
||||
metadataFilePatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`^v\d+\.metadata\.json$`), // Table metadata: v1.metadata.json, v2.metadata.json
|
||||
regexp.MustCompile(`^snap-\d+-\d+-` + uuidPattern + `\.avro$`), // Snapshot manifests: snap-123-1-uuid.avro
|
||||
regexp.MustCompile(`^` + uuidPattern + `-m\d+\.avro$`), // Manifest files: uuid-m0.avro
|
||||
regexp.MustCompile(`^` + uuidPattern + `\.avro$`), // General manifest files
|
||||
regexp.MustCompile(`^version-hint\.text$`), // Version hint file
|
||||
regexp.MustCompile(`^` + uuidPattern + `\.metadata\.json$`), // UUID-named metadata
|
||||
}
|
||||
|
||||
// Patterns for valid data files
|
||||
dataFilePatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`^[^/]+\.parquet$`), // Parquet files
|
||||
regexp.MustCompile(`^[^/]+\.orc$`), // ORC files
|
||||
regexp.MustCompile(`^[^/]+\.avro$`), // Avro files
|
||||
}
|
||||
|
||||
// Data file partition path pattern (e.g., year=2024/month=01/)
|
||||
partitionPathPattern = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*=[^/]+$`)
|
||||
|
||||
// Pattern for valid subdirectory names (alphanumeric, underscore, hyphen, and UUID-style directories)
|
||||
validSubdirectoryPattern = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`)
|
||||
)
|
||||
|
||||
// IcebergLayoutValidator validates that files conform to Iceberg table layout
|
||||
type IcebergLayoutValidator struct{}
|
||||
|
||||
// NewIcebergLayoutValidator creates a new Iceberg layout validator
|
||||
func NewIcebergLayoutValidator() *IcebergLayoutValidator {
|
||||
return &IcebergLayoutValidator{}
|
||||
}
|
||||
|
||||
// ValidateFilePath validates that a file path conforms to Iceberg layout
|
||||
// The path should be relative to the table root (e.g., "metadata/v1.metadata.json" or "data/file.parquet")
|
||||
func (v *IcebergLayoutValidator) ValidateFilePath(relativePath string) error {
|
||||
// Normalize path separators
|
||||
relativePath = strings.TrimPrefix(relativePath, "/")
|
||||
if relativePath == "" {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "empty file path",
|
||||
}
|
||||
}
|
||||
|
||||
parts := strings.SplitN(relativePath, "/", 2)
|
||||
|
||||
topDir := parts[0]
|
||||
|
||||
// Check if top-level directory is allowed
|
||||
if !icebergAllowedDirs[topDir] {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "files must be placed in 'metadata/' or 'data/' directories",
|
||||
}
|
||||
}
|
||||
|
||||
// If it's just a bare top-level key (no trailing slash and no subpath), reject it
|
||||
if len(parts) == 1 {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "must be a directory (use trailing slash) or contain a subpath",
|
||||
}
|
||||
}
|
||||
|
||||
remainingPath := parts[1]
|
||||
if remainingPath == "" {
|
||||
return nil // allow paths like "data/" or "metadata/"
|
||||
}
|
||||
|
||||
switch topDir {
|
||||
case "metadata":
|
||||
return v.validateMetadataFile(remainingPath)
|
||||
case "data":
|
||||
return v.validateDataFile(remainingPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateDirectoryPath validates intermediate subdirectories in a path
|
||||
// isMetadata indicates if we're in the metadata directory (true) or data directory (false)
|
||||
func validateDirectoryPath(normalizedPath string, isMetadata bool) error {
|
||||
if isMetadata {
|
||||
// For metadata, reject any subdirectories (enforce flat structure under metadata/)
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "metadata directory does not support subdirectories",
|
||||
}
|
||||
}
|
||||
|
||||
// For data, validate each partition or subdirectory segment
|
||||
subdirs := strings.Split(normalizedPath, "/")
|
||||
for _, subdir := range subdirs {
|
||||
if subdir == "" {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "invalid partition or subdirectory format in data path: empty segment",
|
||||
}
|
||||
}
|
||||
// For data, allow both partitions and valid subdirectories
|
||||
if !partitionPathPattern.MatchString(subdir) && !isValidSubdirectory(subdir) {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "invalid partition or subdirectory format in data path",
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateFilePatterns validates a filename against allowed patterns
|
||||
// isMetadata indicates if we're validating metadata files (true) or data files (false)
|
||||
func validateFilePatterns(filename string, isMetadata bool) error {
|
||||
var patterns []*regexp.Regexp
|
||||
var errorMsg string
|
||||
|
||||
if isMetadata {
|
||||
patterns = metadataFilePatterns
|
||||
errorMsg = "invalid metadata file format: must be a valid Iceberg metadata, manifest, or snapshot file"
|
||||
} else {
|
||||
patterns = dataFilePatterns
|
||||
errorMsg = "invalid data file format: must be .parquet, .orc, or .avro"
|
||||
}
|
||||
|
||||
for _, pattern := range patterns {
|
||||
if pattern.MatchString(filename) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: errorMsg,
|
||||
}
|
||||
}
|
||||
|
||||
// validateFile validates files with a unified logic for metadata and data directories
|
||||
// isMetadata indicates whether we're validating metadata files (true) or data files (false)
|
||||
// The logic is:
|
||||
// 1. If path ends with "/", it's a directory - validate all parts and return nil
|
||||
// 2. Otherwise, validate intermediate parts, then check the filename against patterns
|
||||
func (v *IcebergLayoutValidator) validateFile(path string, isMetadata bool) error {
|
||||
// Detect if it's a directory (path ends with "/")
|
||||
if strings.HasSuffix(path, "/") {
|
||||
// Normalize by removing trailing slash
|
||||
normalizedPath := strings.TrimSuffix(path, "/")
|
||||
return validateDirectoryPath(normalizedPath, isMetadata)
|
||||
}
|
||||
|
||||
filename := pathpkg.Base(path)
|
||||
|
||||
// Validate intermediate subdirectories if present
|
||||
// Find if there are intermediate directories by looking for the last slash
|
||||
lastSlash := strings.LastIndex(path, "/")
|
||||
if lastSlash != -1 {
|
||||
dir := path[:lastSlash]
|
||||
if err := validateDirectoryPath(dir, isMetadata); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Check against allowed file patterns
|
||||
err := validateFilePatterns(filename, isMetadata)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Path could be for a directory without a trailing slash, e.g., "data/year=2024"
|
||||
if !isMetadata {
|
||||
if partitionPathPattern.MatchString(filename) || isValidSubdirectory(filename) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// validateMetadataFile validates files in the metadata/ directory
|
||||
// This is a thin wrapper that calls validateFile with isMetadata=true
|
||||
func (v *IcebergLayoutValidator) validateMetadataFile(path string) error {
|
||||
return v.validateFile(path, true)
|
||||
}
|
||||
|
||||
// validateDataFile validates files in the data/ directory
|
||||
// This is a thin wrapper that calls validateFile with isMetadata=false
|
||||
func (v *IcebergLayoutValidator) validateDataFile(path string) error {
|
||||
return v.validateFile(path, false)
|
||||
}
|
||||
|
||||
// isValidSubdirectory checks if a path component is a valid subdirectory name
|
||||
func isValidSubdirectory(name string) bool {
|
||||
// Allow alphanumeric, underscore, hyphen, and UUID-style directories
|
||||
return validSubdirectoryPattern.MatchString(name)
|
||||
}
|
||||
|
||||
// IcebergLayoutError represents an Iceberg layout validation error
|
||||
type IcebergLayoutError struct {
|
||||
Code string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *IcebergLayoutError) Error() string {
|
||||
return e.Message
|
||||
}
|
||||
|
||||
// Error code for Iceberg layout violations
|
||||
const (
|
||||
ErrCodeInvalidIcebergLayout = "InvalidIcebergLayout"
|
||||
)
|
||||
|
||||
// TableBucketFileValidator validates file uploads to table buckets
|
||||
type TableBucketFileValidator struct {
|
||||
layoutValidator *IcebergLayoutValidator
|
||||
}
|
||||
|
||||
// NewTableBucketFileValidator creates a new table bucket file validator
|
||||
func NewTableBucketFileValidator() *TableBucketFileValidator {
|
||||
return &TableBucketFileValidator{
|
||||
layoutValidator: NewIcebergLayoutValidator(),
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateTableBucketUpload checks if a file upload to a table bucket conforms to Iceberg layout
|
||||
// fullPath is the complete filer path (e.g., /table-buckets/mybucket/mynamespace/mytable/data/file.parquet)
|
||||
// Returns nil if the path is not a table bucket path or if validation passes
|
||||
// Returns an error if the file doesn't conform to Iceberg layout
|
||||
func (v *TableBucketFileValidator) ValidateTableBucketUpload(fullPath string) error {
|
||||
// Check if this is a table bucket path
|
||||
if !strings.HasPrefix(fullPath, TablesPath+"/") {
|
||||
return nil // Not a table bucket, no validation needed
|
||||
}
|
||||
|
||||
// Extract the path relative to table bucket root
|
||||
// Format: /table-buckets/{bucket}/{namespace}/{table}/{relative-path}
|
||||
relativePath := strings.TrimPrefix(fullPath, TablesPath+"/")
|
||||
parts := strings.SplitN(relativePath, "/", 4)
|
||||
|
||||
// Need at least bucket/namespace/table/file
|
||||
if len(parts) < 4 {
|
||||
// Creating bucket, namespace, or table directories - allow only if preceding parts are non-empty
|
||||
for i := 0; i < len(parts); i++ {
|
||||
if parts[i] == "" {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "bucket, namespace, and table segments cannot be empty",
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// For full paths, also verify bucket, namespace, and table segments are non-empty
|
||||
if parts[0] == "" || parts[1] == "" || parts[2] == "" {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "bucket, namespace, and table segments cannot be empty",
|
||||
}
|
||||
}
|
||||
|
||||
// The last part is the path within the table (data/file.parquet or metadata/v1.json)
|
||||
tableRelativePath := parts[3]
|
||||
if tableRelativePath == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reject paths with empty segments (double slashes) within the table path
|
||||
if strings.HasPrefix(tableRelativePath, "/") || strings.Contains(tableRelativePath, "//") {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "bucket, namespace, and table segments cannot be empty",
|
||||
}
|
||||
}
|
||||
|
||||
return v.layoutValidator.ValidateFilePath(tableRelativePath)
|
||||
}
|
||||
|
||||
// IsTableBucketPath checks if a path is under the table-buckets directory
|
||||
func IsTableBucketPath(fullPath string) bool {
|
||||
return strings.HasPrefix(fullPath, TablesPath+"/")
|
||||
}
|
||||
|
||||
// GetTableInfoFromPath extracts bucket, namespace, and table names from a table bucket path
|
||||
// Returns empty strings if the path doesn't contain enough components
|
||||
func GetTableInfoFromPath(fullPath string) (bucket, namespace, table string) {
|
||||
if !strings.HasPrefix(fullPath, TablesPath+"/") {
|
||||
return "", "", ""
|
||||
}
|
||||
|
||||
relativePath := strings.TrimPrefix(fullPath, TablesPath+"/")
|
||||
parts := strings.SplitN(relativePath, "/", 4)
|
||||
|
||||
if len(parts) >= 1 {
|
||||
bucket = parts[0]
|
||||
}
|
||||
if len(parts) >= 2 {
|
||||
namespace = parts[1]
|
||||
}
|
||||
if len(parts) >= 3 {
|
||||
table = parts[2]
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// ValidateTableBucketUploadWithClient validates upload and checks that the table exists and is ICEBERG format
|
||||
func (v *TableBucketFileValidator) ValidateTableBucketUploadWithClient(
|
||||
ctx context.Context,
|
||||
client filer_pb.SeaweedFilerClient,
|
||||
fullPath string,
|
||||
) error {
|
||||
// First check basic layout
|
||||
if err := v.ValidateTableBucketUpload(fullPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If not a table bucket path, nothing more to check
|
||||
if !IsTableBucketPath(fullPath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get table info and verify it exists
|
||||
bucket, namespace, table := GetTableInfoFromPath(fullPath)
|
||||
if bucket == "" || namespace == "" || table == "" {
|
||||
return nil // Not deep enough to need validation
|
||||
}
|
||||
|
||||
// Verify the table exists and has ICEBERG format by checking its metadata
|
||||
tablePath := getTablePath(bucket, namespace, table)
|
||||
dir, name := splitPath(tablePath)
|
||||
|
||||
resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: dir,
|
||||
Name: name,
|
||||
})
|
||||
if err != nil {
|
||||
// Distinguish between "not found" and other errors
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "table does not exist",
|
||||
}
|
||||
}
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "failed to verify table existence: " + err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
// Check if table has metadata indicating ICEBERG format
|
||||
if resp.Entry == nil || resp.Entry.Extended == nil {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "table is not a valid ICEBERG table (missing metadata)",
|
||||
}
|
||||
}
|
||||
|
||||
metadataBytes, ok := resp.Entry.Extended[ExtendedKeyMetadata]
|
||||
if !ok {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "table is not in ICEBERG format (missing format metadata)",
|
||||
}
|
||||
}
|
||||
|
||||
var metadata tableMetadataInternal
|
||||
if err := json.Unmarshal(metadataBytes, &metadata); err != nil {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "failed to parse table metadata: " + err.Error(),
|
||||
}
|
||||
}
|
||||
const TableFormatIceberg = "ICEBERG"
|
||||
if metadata.Format != TableFormatIceberg {
|
||||
return &IcebergLayoutError{
|
||||
Code: ErrCodeInvalidIcebergLayout,
|
||||
Message: "table is not in " + TableFormatIceberg + " format",
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIcebergLayoutValidator_ValidateFilePath(t *testing.T) {
|
||||
v := NewIcebergLayoutValidator()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
wantErr bool
|
||||
}{
|
||||
// Valid metadata files
|
||||
{"valid metadata v1", "metadata/v1.metadata.json", false},
|
||||
{"valid metadata v123", "metadata/v123.metadata.json", false},
|
||||
{"valid snapshot manifest", "metadata/snap-123-1-abc12345-1234-5678-9abc-def012345678.avro", false},
|
||||
{"valid manifest file", "metadata/abc12345-1234-5678-9abc-def012345678-m0.avro", false},
|
||||
{"valid general manifest", "metadata/abc12345-1234-5678-9abc-def012345678.avro", false},
|
||||
{"valid version hint", "metadata/version-hint.text", false},
|
||||
{"valid uuid metadata", "metadata/abc12345-1234-5678-9abc-def012345678.metadata.json", false},
|
||||
|
||||
// Valid data files
|
||||
{"valid parquet file", "data/file.parquet", false},
|
||||
{"valid orc file", "data/file.orc", false},
|
||||
{"valid avro data file", "data/file.avro", false},
|
||||
{"valid parquet with path", "data/00000-0-abc12345.parquet", false},
|
||||
|
||||
// Valid partitioned data
|
||||
{"valid partitioned parquet", "data/year=2024/file.parquet", false},
|
||||
{"valid multi-partition", "data/year=2024/month=01/file.parquet", false},
|
||||
{"valid bucket subdirectory", "data/bucket0/file.parquet", false},
|
||||
|
||||
// Directories only
|
||||
{"metadata directory bare", "metadata", true},
|
||||
{"data directory bare", "data", true},
|
||||
{"metadata directory with slash", "metadata/", false},
|
||||
{"data directory with slash", "data/", false},
|
||||
|
||||
// Invalid paths
|
||||
{"empty path", "", true},
|
||||
{"invalid top dir", "invalid/file.parquet", true},
|
||||
{"root file", "file.parquet", true},
|
||||
{"invalid metadata file", "metadata/random.txt", true},
|
||||
{"nested metadata directory", "metadata/nested/v1.metadata.json", true},
|
||||
{"nested metadata directory no file", "metadata/nested/", true},
|
||||
{"metadata subdir no slash", "metadata/nested", true},
|
||||
{"invalid data file", "data/file.csv", true},
|
||||
{"invalid data file json", "data/file.json", true},
|
||||
|
||||
// Partition/subdirectory without trailing slashes
|
||||
{"partition directory no slash", "data/year=2024", false},
|
||||
{"data subdirectory no slash", "data/my_subdir", false},
|
||||
{"multi-level partition", "data/event_date=2025-01-01/hour=00/file.parquet", false},
|
||||
{"multi-level partition directory", "data/event_date=2025-01-01/hour=00/", false},
|
||||
{"multi-level partition directory no slash", "data/event_date=2025-01-01/hour=00", false},
|
||||
|
||||
// Double slashes
|
||||
{"data double slash", "data//file.parquet", true},
|
||||
{"data redundant slash", "data/year=2024//file.parquet", true},
|
||||
{"metadata redundant slash", "metadata//v1.metadata.json", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := v.ValidateFilePath(tt.path)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidateFilePath(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIcebergLayoutValidator_PartitionPaths(t *testing.T) {
|
||||
v := NewIcebergLayoutValidator()
|
||||
|
||||
validPaths := []string{
|
||||
"data/year=2024/file.parquet",
|
||||
"data/date=2024-01-15/file.parquet",
|
||||
"data/category=electronics/file.parquet",
|
||||
"data/user_id=12345/file.parquet",
|
||||
"data/region=us-east-1/file.parquet",
|
||||
"data/year=2024/month=01/day=15/file.parquet",
|
||||
}
|
||||
|
||||
for _, path := range validPaths {
|
||||
if err := v.ValidateFilePath(path); err != nil {
|
||||
t.Errorf("ValidateFilePath(%q) should be valid, got error: %v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTableBucketFileValidator_ValidateTableBucketUpload(t *testing.T) {
|
||||
v := NewTableBucketFileValidator()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
wantErr bool
|
||||
}{
|
||||
// Non-table bucket paths should pass (no validation)
|
||||
{"regular bucket path", "/buckets/mybucket/file.txt", false},
|
||||
{"filer path", "/home/user/file.txt", false},
|
||||
|
||||
// Table bucket structure paths (creating directories)
|
||||
{"table bucket root", "/table-buckets/mybucket", false},
|
||||
{"namespace dir", "/table-buckets/mybucket/myns", false},
|
||||
{"table dir", "/table-buckets/mybucket/myns/mytable", false},
|
||||
{"table dir trailing slash", "/table-buckets/mybucket/myns/mytable/", false},
|
||||
|
||||
// Valid table bucket file uploads
|
||||
{"valid parquet upload", "/table-buckets/mybucket/myns/mytable/data/file.parquet", false},
|
||||
{"valid metadata upload", "/table-buckets/mybucket/myns/mytable/metadata/v1.metadata.json", false},
|
||||
{"valid partitioned data", "/table-buckets/mybucket/myns/mytable/data/year=2024/file.parquet", false},
|
||||
|
||||
// Invalid table bucket file uploads
|
||||
{"invalid file type", "/table-buckets/mybucket/myns/mytable/data/file.csv", true},
|
||||
{"invalid top-level dir", "/table-buckets/mybucket/myns/mytable/invalid/file.parquet", true},
|
||||
{"root file in table", "/table-buckets/mybucket/myns/mytable/file.parquet", true},
|
||||
|
||||
// Empty segment cases
|
||||
{"empty bucket", "/table-buckets//myns/mytable/data/file.parquet", true},
|
||||
{"empty namespace", "/table-buckets/mybucket//mytable/data/file.parquet", true},
|
||||
{"empty table", "/table-buckets/mybucket/myns//data/file.parquet", true},
|
||||
{"empty bucket dir", "/table-buckets//", true},
|
||||
{"empty namespace dir", "/table-buckets/mybucket//", true},
|
||||
{"table double slash bypass", "/table-buckets/mybucket/myns/mytable//data/file.parquet", true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := v.ValidateTableBucketUpload(tt.path)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("ValidateTableBucketUpload(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTableBucketPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
{"/table-buckets/mybucket", true},
|
||||
{"/table-buckets/mybucket/ns/table/data/file.parquet", true},
|
||||
{"/buckets/mybucket", false},
|
||||
{"/home/user/file.txt", false},
|
||||
{"table-buckets/mybucket", false}, // missing leading slash
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
if got := IsTableBucketPath(tt.path); got != tt.want {
|
||||
t.Errorf("IsTableBucketPath(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTableInfoFromPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
path string
|
||||
wantBucket string
|
||||
wantNamespace string
|
||||
wantTable string
|
||||
}{
|
||||
{"/table-buckets/mybucket/myns/mytable/data/file.parquet", "mybucket", "myns", "mytable"},
|
||||
{"/table-buckets/mybucket/myns/mytable", "mybucket", "myns", "mytable"},
|
||||
{"/table-buckets/mybucket/myns", "mybucket", "myns", ""},
|
||||
{"/table-buckets/mybucket", "mybucket", "", ""},
|
||||
{"/buckets/mybucket", "", "", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
bucket, namespace, table := GetTableInfoFromPath(tt.path)
|
||||
if bucket != tt.wantBucket || namespace != tt.wantNamespace || table != tt.wantTable {
|
||||
t.Errorf("GetTableInfoFromPath(%q) = (%q, %q, %q), want (%q, %q, %q)",
|
||||
tt.path, bucket, namespace, table, tt.wantBucket, tt.wantNamespace, tt.wantTable)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// Manager provides reusable S3 Tables operations for shell/admin without HTTP routing.
|
||||
type Manager struct {
|
||||
handler *S3TablesHandler
|
||||
}
|
||||
|
||||
// NewManager creates a new Manager.
|
||||
func NewManager() *Manager {
|
||||
return &Manager{handler: NewS3TablesHandler()}
|
||||
}
|
||||
|
||||
// SetRegion sets the AWS region for ARN generation.
|
||||
func (m *Manager) SetRegion(region string) {
|
||||
m.handler.SetRegion(region)
|
||||
}
|
||||
|
||||
// SetAccountID sets the AWS account ID for ARN generation.
|
||||
func (m *Manager) SetAccountID(accountID string) {
|
||||
m.handler.SetAccountID(accountID)
|
||||
}
|
||||
|
||||
// Execute runs an S3 Tables operation and decodes the response into resp (if provided).
|
||||
func (m *Manager) Execute(ctx context.Context, filerClient FilerClient, operation string, req interface{}, resp interface{}, identity string) error {
|
||||
body, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, "/", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
httpReq.Header.Set("Content-Type", "application/x-amz-json-1.1")
|
||||
httpReq.Header.Set("X-Amz-Target", "S3Tables."+operation)
|
||||
if identity != "" {
|
||||
httpReq.Header.Set(s3_constants.AmzAccountId, identity)
|
||||
httpReq = httpReq.WithContext(s3_constants.SetIdentityNameInContext(httpReq.Context(), identity))
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
m.handler.HandleRequest(recorder, httpReq, filerClient)
|
||||
return decodeS3TablesHTTPResponse(recorder, resp)
|
||||
}
|
||||
|
||||
func decodeS3TablesHTTPResponse(recorder *httptest.ResponseRecorder, resp interface{}) error {
|
||||
result := recorder.Result()
|
||||
defer result.Body.Close()
|
||||
data, err := io.ReadAll(result.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if result.StatusCode >= http.StatusBadRequest {
|
||||
var errResp S3TablesError
|
||||
if len(data) > 0 {
|
||||
if jsonErr := json.Unmarshal(data, &errResp); jsonErr == nil && (errResp.Type != "" || errResp.Message != "") {
|
||||
return &errResp
|
||||
}
|
||||
}
|
||||
return &S3TablesError{Type: ErrCodeInternalError, Message: string(bytes.TrimSpace(data))}
|
||||
}
|
||||
if resp == nil || len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := json.Unmarshal(data, resp); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ManagerClient adapts a SeaweedFilerClient to the FilerClient interface.
|
||||
type ManagerClient struct {
|
||||
client filer_pb.SeaweedFilerClient
|
||||
}
|
||||
|
||||
// NewManagerClient wraps a filer client.
|
||||
func NewManagerClient(client filer_pb.SeaweedFilerClient) *ManagerClient {
|
||||
return &ManagerClient{client: client}
|
||||
}
|
||||
|
||||
// WithFilerClient implements FilerClient.
|
||||
func (m *ManagerClient) WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error {
|
||||
if m.client == nil {
|
||||
return errors.New("nil filer client")
|
||||
}
|
||||
return fn(m.client)
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// Permission represents a specific action permission
|
||||
type Permission string
|
||||
|
||||
// IAM Policy structures for evaluation
|
||||
type PolicyDocument struct {
|
||||
Version string `json:"Version"`
|
||||
Statement []Statement `json:"Statement"`
|
||||
}
|
||||
|
||||
// UnmarshalJSON handles both single statement object and array of statements
|
||||
// AWS allows {"Statement": {...}} or {"Statement": [{...}]}
|
||||
func (pd *PolicyDocument) UnmarshalJSON(data []byte) error {
|
||||
type Alias PolicyDocument
|
||||
aux := &struct {
|
||||
Statement interface{} `json:"Statement"`
|
||||
*Alias
|
||||
}{
|
||||
Alias: (*Alias)(pd),
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &aux); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Handle Statement as either a single object or array
|
||||
switch s := aux.Statement.(type) {
|
||||
case map[string]interface{}:
|
||||
// Single statement object - unmarshal to one Statement
|
||||
stmtData, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal single statement: %w", err)
|
||||
}
|
||||
var stmt Statement
|
||||
if err := json.Unmarshal(stmtData, &stmt); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal single statement: %w", err)
|
||||
}
|
||||
pd.Statement = []Statement{stmt}
|
||||
case []interface{}:
|
||||
// Array of statements - normal handling
|
||||
stmtData, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal statement array: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(stmtData, &pd.Statement); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal statement array: %w", err)
|
||||
}
|
||||
case nil:
|
||||
// No statements
|
||||
pd.Statement = []Statement{}
|
||||
default:
|
||||
return fmt.Errorf("Statement must be an object or array, got %T", aux.Statement)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type Statement struct {
|
||||
Effect string `json:"Effect"` // "Allow" or "Deny"
|
||||
Principal interface{} `json:"Principal"` // Can be string, []string, or map
|
||||
Action interface{} `json:"Action"` // Can be string or []string
|
||||
Resource interface{} `json:"Resource"` // Can be string or []string
|
||||
Condition map[string]map[string]interface{} `json:"Condition,omitempty"`
|
||||
}
|
||||
|
||||
type PolicyContext struct {
|
||||
Namespace string
|
||||
TableName string
|
||||
TableBucketName string
|
||||
IdentityActions []string
|
||||
RequestTags map[string]string
|
||||
ResourceTags map[string]string
|
||||
TableBucketTags map[string]string
|
||||
TagKeys []string
|
||||
SSEAlgorithm string
|
||||
KMSKeyArn string
|
||||
StorageClass string
|
||||
}
|
||||
|
||||
// CheckPermissionWithResource checks if a principal has permission to perform an operation on a specific resource
|
||||
func CheckPermissionWithResource(operation, principal, owner, resourcePolicy, resourceARN string) bool {
|
||||
return CheckPermissionWithContext(operation, principal, owner, resourcePolicy, resourceARN, nil)
|
||||
}
|
||||
|
||||
// CheckPermission checks if a principal has permission to perform an operation
|
||||
// (without resource-specific validation - for backward compatibility)
|
||||
func CheckPermission(operation, principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermissionWithContext(operation, principal, owner, resourcePolicy, "", nil)
|
||||
}
|
||||
|
||||
// CheckPermissionWithContext checks permission with optional resource and condition context.
|
||||
func CheckPermissionWithContext(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
|
||||
// Deny access if identities are empty
|
||||
if principal == "" || owner == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Admin always has permission.
|
||||
if principal == s3_constants.AccountAdminId {
|
||||
return true
|
||||
}
|
||||
|
||||
return checkPermission(operation, principal, owner, resourcePolicy, resourceARN, ctx)
|
||||
}
|
||||
|
||||
func checkPermission(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
|
||||
// Owner always has permission
|
||||
if principal == owner {
|
||||
return true
|
||||
}
|
||||
|
||||
if hasIdentityPermission(operation, ctx) {
|
||||
return true
|
||||
}
|
||||
|
||||
// If no policy is provided, deny access (default deny)
|
||||
if resourcePolicy == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
// Normalize operation to full IAM-style action name (e.g., "s3tables:CreateTableBucket")
|
||||
// if not already prefixed
|
||||
fullAction := operation
|
||||
if !strings.Contains(operation, ":") {
|
||||
fullAction = "s3tables:" + operation
|
||||
}
|
||||
|
||||
// Parse and evaluate policy
|
||||
var policy PolicyDocument
|
||||
if err := json.Unmarshal([]byte(resourcePolicy), &policy); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Evaluate policy statements
|
||||
// Default is deny, so we need an explicit allow
|
||||
hasAllow := false
|
||||
|
||||
for _, stmt := range policy.Statement {
|
||||
// Check if principal matches
|
||||
if !matchesPrincipal(stmt.Principal, principal) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if action matches (using normalized full action name)
|
||||
if !matchesAction(stmt.Action, fullAction) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if resource matches (if resourceARN specified and Resource field exists)
|
||||
if resourceARN != "" && !matchesResource(stmt.Resource, resourceARN) {
|
||||
continue
|
||||
}
|
||||
|
||||
if !matchesConditions(stmt.Condition, ctx) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Statement matches - check effect
|
||||
if stmt.Effect == "Allow" {
|
||||
hasAllow = true
|
||||
} else if stmt.Effect == "Deny" {
|
||||
// Explicit deny always wins
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return hasAllow
|
||||
}
|
||||
|
||||
func hasIdentityPermission(operation string, ctx *PolicyContext) bool {
|
||||
if ctx == nil || len(ctx.IdentityActions) == 0 {
|
||||
return false
|
||||
}
|
||||
fullAction := operation
|
||||
if !strings.Contains(operation, ":") {
|
||||
fullAction = "s3tables:" + operation
|
||||
}
|
||||
candidates := []string{operation, fullAction}
|
||||
if ctx.TableBucketName != "" {
|
||||
candidates = append(candidates, operation+":"+ctx.TableBucketName, fullAction+":"+ctx.TableBucketName)
|
||||
}
|
||||
for _, action := range ctx.IdentityActions {
|
||||
for _, candidate := range candidates {
|
||||
if action == candidate {
|
||||
return true
|
||||
}
|
||||
if strings.ContainsAny(action, "*?") && policy_engine.MatchesWildcard(action, candidate) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// matchesPrincipal checks if the principal matches the statement's principal
|
||||
func matchesPrincipal(principalSpec interface{}, principal string) bool {
|
||||
if principalSpec == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
switch p := principalSpec.(type) {
|
||||
case string:
|
||||
// Direct string match or wildcard
|
||||
if p == "*" || p == principal {
|
||||
return true
|
||||
}
|
||||
// Support wildcard matching for principals (e.g., "arn:aws:iam::*:user/admin")
|
||||
return policy_engine.MatchesWildcard(p, principal)
|
||||
case []interface{}:
|
||||
// Array of principals
|
||||
for _, item := range p {
|
||||
if str, ok := item.(string); ok {
|
||||
if str == "*" || str == principal {
|
||||
return true
|
||||
}
|
||||
// Support wildcard matching
|
||||
if policy_engine.MatchesWildcard(str, principal) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
case map[string]interface{}:
|
||||
// AWS-style principal with service prefix, e.g., {"AWS": "arn:aws:iam::..."}
|
||||
// For S3 Tables, we primarily care about the AWS key
|
||||
if aws, ok := p["AWS"]; ok {
|
||||
return matchesPrincipal(aws, principal)
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// matchesAction checks if the action matches the statement's action
|
||||
func matchesAction(actionSpec interface{}, action string) bool {
|
||||
if actionSpec == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
switch a := actionSpec.(type) {
|
||||
case string:
|
||||
// Direct match or wildcard
|
||||
return matchesActionPattern(a, action)
|
||||
case []interface{}:
|
||||
// Array of actions
|
||||
for _, item := range a {
|
||||
if str, ok := item.(string); ok {
|
||||
if matchesActionPattern(str, action) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// matchesActionPattern checks if an action matches a pattern (supports wildcards)
|
||||
// This uses the policy_engine.MatchesWildcard function for full wildcard support,
|
||||
// including middle wildcards (e.g., "s3tables:Get*Table") for complete IAM compatibility.
|
||||
func matchesActionPattern(pattern, action string) bool {
|
||||
if pattern == "*" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Exact match
|
||||
if pattern == action {
|
||||
return true
|
||||
}
|
||||
|
||||
// Wildcard match using policy engine's wildcard matcher
|
||||
// Supports both * (any sequence) and ? (single character) anywhere in the pattern
|
||||
return policy_engine.MatchesWildcard(pattern, action)
|
||||
}
|
||||
|
||||
func matchesConditions(conditions map[string]map[string]interface{}, ctx *PolicyContext) bool {
|
||||
if len(conditions) == 0 {
|
||||
return true
|
||||
}
|
||||
if ctx == nil {
|
||||
return false
|
||||
}
|
||||
for operator, conditionValues := range conditions {
|
||||
if !matchesConditionOperator(operator, conditionValues, ctx) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func matchesConditionOperator(operator string, conditionValues map[string]interface{}, ctx *PolicyContext) bool {
|
||||
evaluator, err := policy_engine.GetConditionEvaluator(operator)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
for key, value := range conditionValues {
|
||||
contextVals := getConditionContextValues(key, ctx)
|
||||
if !evaluator.Evaluate(value, contextVals) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func getConditionContextValues(key string, ctx *PolicyContext) []string {
|
||||
switch key {
|
||||
case "s3tables:namespace":
|
||||
return []string{ctx.Namespace}
|
||||
case "s3tables:tableName":
|
||||
return []string{ctx.TableName}
|
||||
case "s3tables:tableBucketName":
|
||||
return []string{ctx.TableBucketName}
|
||||
case "s3tables:SSEAlgorithm":
|
||||
return []string{ctx.SSEAlgorithm}
|
||||
case "s3tables:KMSKeyArn":
|
||||
return []string{ctx.KMSKeyArn}
|
||||
case "s3tables:StorageClass":
|
||||
return []string{ctx.StorageClass}
|
||||
case "aws:TagKeys":
|
||||
return ctx.TagKeys
|
||||
}
|
||||
if strings.HasPrefix(key, "aws:RequestTag/") {
|
||||
tagKey := strings.TrimPrefix(key, "aws:RequestTag/")
|
||||
if val, ok := ctx.RequestTags[tagKey]; ok {
|
||||
return []string{val}
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(key, "aws:ResourceTag/") {
|
||||
tagKey := strings.TrimPrefix(key, "aws:ResourceTag/")
|
||||
if val, ok := ctx.ResourceTags[tagKey]; ok {
|
||||
return []string{val}
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(key, "s3tables:TableBucketTag/") {
|
||||
tagKey := strings.TrimPrefix(key, "s3tables:TableBucketTag/")
|
||||
if val, ok := ctx.TableBucketTags[tagKey]; ok {
|
||||
return []string{val}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// matchesResource checks if the resource ARN matches the statement's resource specification
|
||||
// Returns true if resource matches or if Resource is not specified (implicit match)
|
||||
func matchesResource(resourceSpec interface{}, resourceARN string) bool {
|
||||
// If no Resource is specified, match all resources (implicit *)
|
||||
if resourceSpec == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
switch r := resourceSpec.(type) {
|
||||
case string:
|
||||
// Direct match or wildcard
|
||||
return matchesResourcePattern(r, resourceARN)
|
||||
case []interface{}:
|
||||
// Array of resources - match if any matches
|
||||
for _, item := range r {
|
||||
if str, ok := item.(string); ok {
|
||||
if matchesResourcePattern(str, resourceARN) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// matchesResourcePattern checks if a resource ARN matches a pattern (supports wildcards)
|
||||
func matchesResourcePattern(pattern, resourceARN string) bool {
|
||||
if pattern == "*" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Exact match
|
||||
if pattern == resourceARN {
|
||||
return true
|
||||
}
|
||||
|
||||
// Wildcard match using policy engine's wildcard matcher
|
||||
return policy_engine.MatchesWildcard(pattern, resourceARN)
|
||||
}
|
||||
|
||||
// Helper functions for specific permissions
|
||||
|
||||
// CanCreateTableBucket checks if principal can create table buckets
|
||||
func CanCreateTableBucket(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("CreateTableBucket", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanGetTableBucket checks if principal can get table bucket details
|
||||
func CanGetTableBucket(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("GetTableBucket", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanListTableBuckets checks if principal can list table buckets
|
||||
func CanListTableBuckets(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("ListTableBuckets", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanDeleteTableBucket checks if principal can delete table buckets
|
||||
func CanDeleteTableBucket(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("DeleteTableBucket", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanPutTableBucketPolicy checks if principal can put table bucket policies
|
||||
func CanPutTableBucketPolicy(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("PutTableBucketPolicy", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanGetTableBucketPolicy checks if principal can get table bucket policies
|
||||
func CanGetTableBucketPolicy(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("GetTableBucketPolicy", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanDeleteTableBucketPolicy checks if principal can delete table bucket policies
|
||||
func CanDeleteTableBucketPolicy(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("DeleteTableBucketPolicy", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanCreateNamespace checks if principal can create namespaces
|
||||
func CanCreateNamespace(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("CreateNamespace", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanGetNamespace checks if principal can get namespace details
|
||||
func CanGetNamespace(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("GetNamespace", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanListNamespaces checks if principal can list namespaces
|
||||
func CanListNamespaces(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("ListNamespaces", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanDeleteNamespace checks if principal can delete namespaces
|
||||
func CanDeleteNamespace(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("DeleteNamespace", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanCreateTable checks if principal can create tables
|
||||
func CanCreateTable(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("CreateTable", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanGetTable checks if principal can get table details
|
||||
func CanGetTable(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("GetTable", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanListTables checks if principal can list tables
|
||||
func CanListTables(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("ListTables", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanDeleteTable checks if principal can delete tables
|
||||
func CanDeleteTable(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("DeleteTable", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanPutTablePolicy checks if principal can put table policies
|
||||
func CanPutTablePolicy(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("PutTablePolicy", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanGetTablePolicy checks if principal can get table policies
|
||||
func CanGetTablePolicy(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("GetTablePolicy", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanDeleteTablePolicy checks if principal can delete table policies
|
||||
func CanDeleteTablePolicy(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("DeleteTablePolicy", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanTagResource checks if principal can tag a resource
|
||||
func CanTagResource(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("TagResource", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanUntagResource checks if principal can untag a resource
|
||||
func CanUntagResource(principal, owner, resourcePolicy string) bool {
|
||||
return CheckPermission("UntagResource", principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// CanManageTags checks if principal can manage tags (tag or untag)
|
||||
func CanManageTags(principal, owner, resourcePolicy string) bool {
|
||||
return CanTagResource(principal, owner, resourcePolicy) || CanUntagResource(principal, owner, resourcePolicy)
|
||||
}
|
||||
|
||||
// AuthError represents an authorization error
|
||||
type AuthError struct {
|
||||
Operation string
|
||||
Principal string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *AuthError) Error() string {
|
||||
return "unauthorized: " + e.Principal + " is not permitted to perform " + e.Operation + ": " + e.Message
|
||||
}
|
||||
|
||||
// NewAuthError creates a new authorization error
|
||||
func NewAuthError(operation, principal, message string) *AuthError {
|
||||
return &AuthError{
|
||||
Operation: operation,
|
||||
Principal: principal,
|
||||
Message: message,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package s3tables
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMatchesActionPattern(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pattern string
|
||||
action string
|
||||
expected bool
|
||||
}{
|
||||
// Exact matches
|
||||
{"exact match", "GetTable", "GetTable", true},
|
||||
{"no match", "GetTable", "DeleteTable", false},
|
||||
|
||||
// Universal wildcard
|
||||
{"universal wildcard", "*", "anything", true},
|
||||
|
||||
// Suffix wildcards
|
||||
{"suffix wildcard match", "s3tables:*", "s3tables:GetTable", true},
|
||||
{"suffix wildcard no match", "s3tables:*", "iam:GetUser", false},
|
||||
|
||||
// Middle wildcards (new capability from policy_engine)
|
||||
{"middle wildcard Get*Table", "s3tables:Get*Table", "s3tables:GetTable", true},
|
||||
{"middle wildcard Get*Table no match GetTableBucket", "s3tables:Get*Table", "s3tables:GetTableBucket", false},
|
||||
{"middle wildcard Get*Table no match DeleteTable", "s3tables:Get*Table", "s3tables:DeleteTable", false},
|
||||
{"middle wildcard *Table*", "s3tables:*Table*", "s3tables:GetTableBucket", true},
|
||||
{"middle wildcard *Table* match CreateTable", "s3tables:*Table*", "s3tables:CreateTable", true},
|
||||
|
||||
// Question mark wildcards
|
||||
{"question mark single char", "GetTable?", "GetTableX", true},
|
||||
{"question mark no match", "GetTable?", "GetTableXY", false},
|
||||
|
||||
// Combined wildcards
|
||||
{"combined * and ? singular", "s3tables:Get?able*", "s3tables:GetTable", true},
|
||||
{"combined * and ? plural", "s3tables:Get?able*", "s3tables:GetTables", true},
|
||||
{"combined no match - ? needs 1 char", "s3tables:Get?able*", "s3tables:Getable", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := matchesActionPattern(tt.pattern, tt.action)
|
||||
if result != tt.expected {
|
||||
t.Errorf("matchesActionPattern(%q, %q) = %v, want %v", tt.pattern, tt.action, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesPrincipal(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
principalSpec interface{}
|
||||
principal string
|
||||
expected bool
|
||||
}{
|
||||
// String principals
|
||||
{"exact match", "user123", "user123", true},
|
||||
{"no match", "user123", "user456", false},
|
||||
{"universal wildcard", "*", "anyone", true},
|
||||
|
||||
// Wildcard principals
|
||||
{"prefix wildcard", "arn:aws:iam::123456789012:user/*", "arn:aws:iam::123456789012:user/admin", true},
|
||||
{"prefix wildcard no match", "arn:aws:iam::123456789012:user/*", "arn:aws:iam::987654321098:user/admin", false},
|
||||
{"middle wildcard", "arn:aws:iam::*:user/admin", "arn:aws:iam::123456789012:user/admin", true},
|
||||
|
||||
// Array of principals
|
||||
{"array match first", []interface{}{"user1", "user2"}, "user1", true},
|
||||
{"array match second", []interface{}{"user1", "user2"}, "user2", true},
|
||||
{"array no match", []interface{}{"user1", "user2"}, "user3", false},
|
||||
{"array wildcard", []interface{}{"user1", "arn:aws:iam::*:user/admin"}, "arn:aws:iam::123:user/admin", true},
|
||||
|
||||
// Map-style AWS principals
|
||||
{"AWS map exact", map[string]interface{}{"AWS": "user123"}, "user123", true},
|
||||
{"AWS map wildcard", map[string]interface{}{"AWS": "arn:aws:iam::*:user/admin"}, "arn:aws:iam::123:user/admin", true},
|
||||
{"AWS map array", map[string]interface{}{"AWS": []interface{}{"user1", "user2"}}, "user1", true},
|
||||
|
||||
// Nil/empty cases
|
||||
{"nil principal", nil, "user123", false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := matchesPrincipal(tt.principalSpec, tt.principal)
|
||||
if result != tt.expected {
|
||||
t.Errorf("matchesPrincipal(%v, %q) = %v, want %v", tt.principalSpec, tt.principal, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluatePolicyWithConditions(t *testing.T) {
|
||||
policy := &PolicyDocument{
|
||||
Statement: []Statement{
|
||||
{
|
||||
Effect: "Allow",
|
||||
Principal: "*",
|
||||
Action: "s3tables:GetTable",
|
||||
Condition: map[string]map[string]interface{}{
|
||||
"StringEquals": {
|
||||
"s3tables:namespace": "default",
|
||||
},
|
||||
"StringLike": {
|
||||
"s3tables:tableName": "test_*",
|
||||
},
|
||||
"NumericGreaterThan": {
|
||||
"aws:RequestTag/priority": "10",
|
||||
},
|
||||
"Bool": {
|
||||
"aws:ResourceTag/is_public": "true",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
policyBytes, _ := json.Marshal(policy)
|
||||
policyStr := string(policyBytes)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ctx *PolicyContext
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
"all conditions match",
|
||||
&PolicyContext{
|
||||
Namespace: "default",
|
||||
TableName: "test_table",
|
||||
RequestTags: map[string]string{
|
||||
"priority": "15",
|
||||
},
|
||||
ResourceTags: map[string]string{
|
||||
"is_public": "true",
|
||||
},
|
||||
},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"namespace mismatch",
|
||||
&PolicyContext{
|
||||
Namespace: "other",
|
||||
TableName: "test_table",
|
||||
RequestTags: map[string]string{
|
||||
"priority": "15",
|
||||
},
|
||||
ResourceTags: map[string]string{
|
||||
"is_public": "true",
|
||||
},
|
||||
},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"table name mismatch",
|
||||
&PolicyContext{
|
||||
Namespace: "default",
|
||||
TableName: "other_table",
|
||||
RequestTags: map[string]string{
|
||||
"priority": "15",
|
||||
},
|
||||
ResourceTags: map[string]string{
|
||||
"is_public": "true",
|
||||
},
|
||||
},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"numeric condition failure",
|
||||
&PolicyContext{
|
||||
Namespace: "default",
|
||||
TableName: "test_table",
|
||||
RequestTags: map[string]string{
|
||||
"priority": "5",
|
||||
},
|
||||
ResourceTags: map[string]string{
|
||||
"is_public": "true",
|
||||
},
|
||||
},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"bool condition failure",
|
||||
&PolicyContext{
|
||||
Namespace: "default",
|
||||
TableName: "test_table",
|
||||
RequestTags: map[string]string{
|
||||
"priority": "15",
|
||||
},
|
||||
ResourceTags: map[string]string{
|
||||
"is_public": "false",
|
||||
},
|
||||
},
|
||||
false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// principal="user123", owner="owner123"
|
||||
result := CheckPermissionWithContext("s3tables:GetTable", "user123", "owner123", policyStr, "", tt.ctx)
|
||||
if result != tt.expected {
|
||||
t.Errorf("CheckPermissionWithContext() = %v, want %v", result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user