mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-10 08:35:50 +00:00
Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7e2d4c24de | ||
|
|
202872d88d | ||
|
|
cacfcf5eb2 | ||
|
|
62669040c6 | ||
|
|
777e36bcf3 | ||
|
|
edfe312216 | ||
|
|
489dddf84e | ||
|
|
232d58dc18 | ||
|
|
b2e4474bf2 | ||
|
|
e5fab95905 | ||
|
|
0e8db10a54 | ||
|
|
0c5068cb28 | ||
|
|
d3dabd2d84 | ||
|
|
3e76104d7e | ||
|
|
a770a4983b | ||
|
|
f2634e6a71 | ||
|
|
c1b190cdd6 | ||
|
|
78dda049e5 | ||
|
|
503b0a283c | ||
|
|
eea5d91acb | ||
|
|
e05994ca7e |
@@ -44,7 +44,7 @@ jobs:
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -109,7 +109,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -224,4 +224,3 @@ jobs:
|
||||
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
|
||||
@@ -134,7 +134,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: output-logs
|
||||
path: docker/output.log
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: ec-integration-test-logs
|
||||
path: test/erasure_coding
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: fuse-integration-test-results
|
||||
path: |
|
||||
|
||||
@@ -53,7 +53,6 @@ jobs:
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS with optimized settings for CI
|
||||
# Include S3 API for s3copier integration tests
|
||||
weed server -dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.electionTimeout=1s \
|
||||
@@ -62,9 +61,6 @@ jobs:
|
||||
-volume.preStopSeconds=1 \
|
||||
-master.peers=none \
|
||||
-filer -filer.maxMB=64 \
|
||||
-s3 -s3.port=8333 \
|
||||
-s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
-master.port=9333 \
|
||||
-volume.port=8080 \
|
||||
-filer.port=8888 \
|
||||
@@ -109,16 +105,6 @@ jobs:
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Wait for S3 API
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8333/ > /dev/null 2>&1; then
|
||||
echo "✓ S3 API is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 API... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "✓ All SeaweedFS components are ready!"
|
||||
|
||||
# Display cluster status
|
||||
@@ -147,20 +133,6 @@ jobs:
|
||||
run: |
|
||||
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Run S3 ETag Validation Tests (Issue #7768)
|
||||
working-directory: other/java/s3copier
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
S3_ACCESS_KEY: some_access_key1
|
||||
S3_SECRET_KEY: some_secret_key1
|
||||
run: |
|
||||
echo "Running S3 ETag validation tests against $S3_ENDPOINT"
|
||||
mvn test -Dtest=ETagValidationTest \
|
||||
-DS3_ENDPOINT=$S3_ENDPOINT \
|
||||
-DS3_ACCESS_KEY=$S3_ACCESS_KEY \
|
||||
-DS3_SECRET_KEY=$S3_SECRET_KEY \
|
||||
-Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Display logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: test-reports-java-${{ matrix.java }}
|
||||
path: |
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: metadata-subscribe-test-logs
|
||||
path: |
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: postgres-logs
|
||||
path: test/postgres/postgres-output.log
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
name: "S3 Filer Group Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-filer-group-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
s3-filer-group-tests:
|
||||
name: S3 Filer Group Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Filer Group Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/filer_group
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Filer Group Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
# The test-with-server target handles server startup/shutdown automatically
|
||||
# Server is started with -filer.group=testgroup
|
||||
make test-with-server || {
|
||||
echo "❌ Filer group tests failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/filer_group
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-filer-group-test-logs
|
||||
path: test/s3/filer_group/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-versioning-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-versioning-compatibility-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-cors-compatibility-logs
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -239,7 +239,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-retention-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -306,7 +306,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-cors-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -355,7 +355,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-retention-worm-logs
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -365,6 +365,8 @@ jobs:
|
||||
name: S3 Versioning Stress Test
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 35
|
||||
# Only run stress tests on master branch pushes to avoid overloading PR testing
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -389,9 +391,6 @@ jobs:
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Start server for stress tests
|
||||
make start-server
|
||||
|
||||
# Run stress tests (concurrent operations)
|
||||
make test-versioning-stress || {
|
||||
echo "❌ Stress test failed, checking logs..."
|
||||
@@ -399,30 +398,14 @@ jobs:
|
||||
echo "=== Server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
make stop-server
|
||||
make clean
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Run pagination stress tests (>1000 versions)
|
||||
echo "=== Running pagination stress tests ==="
|
||||
make test-versioning-pagination-stress || {
|
||||
echo "❌ Pagination stress test failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
make stop-server
|
||||
make clean
|
||||
exit 1
|
||||
}
|
||||
|
||||
make stop-server
|
||||
make clean
|
||||
|
||||
- name: Upload stress test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-versioning-stress-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -476,65 +459,10 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-tagging-test-logs
|
||||
path: test/s3/tagging/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-remote-cache-tests:
|
||||
name: S3 Remote Cache Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Remote Cache Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/remote_cache
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run the remote cache integration tests
|
||||
# Tests singleflight deduplication for caching remote objects
|
||||
make test-with-server || {
|
||||
echo "❌ Test failed, checking logs..."
|
||||
if [ -f primary-weed.log ]; then
|
||||
echo "=== Primary server logs ==="
|
||||
tail -100 primary-weed.log
|
||||
fi
|
||||
if [ -f remote-weed.log ]; then
|
||||
echo "=== Remote server logs ==="
|
||||
tail -100 remote-weed.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-remote-cache-test-logs
|
||||
path: |
|
||||
test/s3/remote_cache/primary-weed.log
|
||||
test/s3/remote_cache/remote-weed.log
|
||||
retention-days: 3
|
||||
|
||||
# Removed SSE-C integration tests and compatibility job
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: iam-unit-test-results
|
||||
path: |
|
||||
@@ -162,7 +162,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-iam-integration-logs-${{ matrix.test-type }}
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -222,7 +222,7 @@ jobs:
|
||||
|
||||
- name: Upload distributed test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-iam-distributed-logs
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -274,7 +274,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-iam-performance-results
|
||||
path: |
|
||||
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-keycloak-test-logs
|
||||
path: |
|
||||
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: test-logs-python-${{ matrix.python-version }}
|
||||
path: |
|
||||
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -143,7 +143,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-compatibility-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-metadata-persistence-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -241,7 +241,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-copy-operations-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -290,7 +290,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-multipart-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -340,7 +340,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-performance-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
|
||||
@@ -91,4 +91,3 @@ jobs:
|
||||
echo "| readonly | /public | Read-only |" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -125,7 +125,7 @@ jobs:
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: spark-test-results
|
||||
path: test/java/spark/target/surefire-reports/
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
- name: Cache Apache Spark
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
id: cache-spark
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: spark-3.5.0-bin-hadoop3
|
||||
key: spark-3.5.0-hadoop3
|
||||
|
||||
@@ -18,7 +18,7 @@ defaults:
|
||||
jobs:
|
||||
awscli-tests:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: seaweedfs-logs
|
||||
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
name: "TUS Protocol Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/tus-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
tus-integration-tests:
|
||||
name: TUS Protocol Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run TUS Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/tus
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting TUS Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
make test-with-server || {
|
||||
echo "TUS integration tests failed, checking logs..."
|
||||
if [ -f /tmp/seaweedfs-tus-filer.log ]; then
|
||||
echo "=== Filer logs ==="
|
||||
tail -100 /tmp/seaweedfs-tus-filer.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs-tus-master.log ]; then
|
||||
echo "=== Master logs ==="
|
||||
tail -50 /tmp/seaweedfs-tus-master.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs-tus-volume.log ]; then
|
||||
echo "=== Volume logs ==="
|
||||
tail -50 /tmp/seaweedfs-tus-volume.log
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/tus
|
||||
run: |
|
||||
echo "=== Filer Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-filer.log ]; then
|
||||
echo "Last 100 lines of filer logs:"
|
||||
tail -100 /tmp/seaweedfs-tus-filer.log
|
||||
else
|
||||
echo "No filer log file found"
|
||||
fi
|
||||
|
||||
echo "=== Master Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-master.log ]; then
|
||||
tail -50 /tmp/seaweedfs-tus-master.log
|
||||
else
|
||||
echo "No master log file found"
|
||||
fi
|
||||
|
||||
echo "=== Volume Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-volume.log ]; then
|
||||
tail -50 /tmp/seaweedfs-tus-volume.log
|
||||
else
|
||||
echo "No volume log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp 2>/dev/null | grep -E "(18888|19333|18080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tus-test-logs
|
||||
path: |
|
||||
/tmp/seaweedfs-tus-*.log
|
||||
retention-days: 3
|
||||
@@ -124,10 +124,3 @@ ADVANCED_IAM_DEVELOPMENT_PLAN.md
|
||||
*.log
|
||||
weed-iam
|
||||
test/kafka/kafka-client-loadtest/weed-linux-arm64
|
||||
/test/tus/filerldb2
|
||||
coverage.out
|
||||
/test/s3/remote_cache/test-primary-data
|
||||
/test/s3/remote_cache/test-remote-data
|
||||
test/s3/remote_cache/remote-server.pid
|
||||
test/s3/remote_cache/primary-server.pid
|
||||
/test/erasure_coding/filerldb2
|
||||
|
||||
@@ -28,7 +28,7 @@ require (
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.15.0
|
||||
github.com/go-redsync/redsync/v4 v4.14.0
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/gocql/gocql v1.7.0
|
||||
@@ -71,7 +71,7 @@ require (
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.6
|
||||
github.com/seaweedfs/raft v1.1.3
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
@@ -96,14 +96,14 @@ require (
|
||||
gocloud.dev v0.43.0
|
||||
gocloud.dev/pubsub/natspubsub v0.43.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.43.0
|
||||
golang.org/x/crypto v0.46.0
|
||||
golang.org/x/crypto v0.45.0
|
||||
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
|
||||
golang.org/x/image v0.34.0
|
||||
golang.org/x/net v0.48.0
|
||||
golang.org/x/image v0.33.0
|
||||
golang.org/x/net v0.47.0
|
||||
golang.org/x/oauth2 v0.32.0 // indirect
|
||||
golang.org/x/sys v0.39.0
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/tools v0.39.0 // indirect
|
||||
golang.org/x/sys v0.38.0
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/tools v0.38.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.247.0
|
||||
google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect
|
||||
@@ -126,9 +126,9 @@ require (
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
@@ -152,14 +152,14 @@ require (
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rclone/rclone v1.71.2
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/redis/go-redis/v9 v9.14.1
|
||||
github.com/schollz/progressbar/v3 v3.18.0
|
||||
github.com/shirou/gopsutil/v4 v4.25.11
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.1
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.6
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.19.0
|
||||
@@ -173,7 +173,6 @@ require (
|
||||
cloud.google.com/go/longrunning v0.6.7 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.0.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
github.com/blevesearch/snowballstem v0.9.0 // indirect
|
||||
@@ -204,8 +203,8 @@ require (
|
||||
github.com/openzipkin/zipkin-go v0.4.3 // indirect
|
||||
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/quic-go/quic-go v0.57.0 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/quic-go/quic-go v0.54.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
@@ -219,9 +218,10 @@ require (
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
|
||||
go.uber.org/mock v0.5.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.30.0 // indirect
|
||||
golang.org/x/mod v0.29.0 // indirect
|
||||
gonum.org/v1/gonum v0.16.0 // indirect
|
||||
)
|
||||
|
||||
@@ -261,21 +261,21 @@ require (
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
@@ -446,7 +446,7 @@ require (
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/term v0.38.0 // indirect
|
||||
golang.org/x/term v0.37.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
|
||||
|
||||
@@ -651,6 +651,8 @@ github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmg
|
||||
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
|
||||
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
|
||||
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad h1:fQBkhYv86zyW95PWhzBlkgz3NoY1ue0L+8oYBaoCMbg=
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20240515141816-262c6fe778ad/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3 h1:WZaTKNHCfcw7fWSR6/RKnCldVzvYZC+Y20Su4lffEIg=
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc h1:LoL75er+LKDHDUfU5tRvFwxH0LjPpZN8OoG8Ll+liGU=
|
||||
@@ -665,48 +667,46 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/bazelbuild/rules_go v0.46.0 h1:CTefzjN/D3Cdn3rkrM6qMWuQj59OBcuOjyIp3m4hZ7s=
|
||||
@@ -988,8 +988,8 @@ github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOr
|
||||
github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-redsync/redsync/v4 v4.15.0 h1:KH/XymuxSV7vyKs6z1Cxxj+N+N18JlPxgXeP6x4JY54=
|
||||
github.com/go-redsync/redsync/v4 v4.15.0/go.mod h1:qNp+lLs3vkfZbtA/aM/OjlZHfEr5YTAYhRktFPKHC7s=
|
||||
github.com/go-redsync/redsync/v4 v4.14.0 h1:zyxzFJsmQHIPBl8iBT7KFKohWsjsghgGLiP8TnFMLNc=
|
||||
github.com/go-redsync/redsync/v4 v4.14.0/go.mod h1:twMlVd19upZ/juvJyJGlQOSQxor1oeHtjs62l4pRFzo=
|
||||
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
|
||||
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
@@ -1072,8 +1072,8 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
|
||||
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/gomodule/redigo v1.9.3 h1:dNPSXeXv6HCq2jdyWfjgmhBdqnR6PRO3m/G05nvpPC8=
|
||||
github.com/gomodule/redigo v1.9.3/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
|
||||
github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s=
|
||||
github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
|
||||
@@ -1565,10 +1565,10 @@ github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4
|
||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
|
||||
github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
|
||||
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
|
||||
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
|
||||
github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg=
|
||||
github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
|
||||
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
|
||||
github.com/rclone/rclone v1.71.2 h1:3Jk5xNPFrZhVABRuN/OPvApuZQddpE2tkhYMuEn1Ud4=
|
||||
@@ -1577,12 +1577,12 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
|
||||
github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.69/go.mod h1:iC4Y8DoN0Uth0Uezg9e2trvNRC7QAgGeuP2OPLb5ccI=
|
||||
github.com/redis/go-redis/v9 v9.14.1 h1:nDCrEiJmfOWhD76xlaw+HXT0c9hfNWeXgl0vIRYSDvQ=
|
||||
github.com/redis/go-redis/v9 v9.14.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
|
||||
github.com/redis/rueidis v1.0.64 h1:XqgbueDuNV3qFdVdQwAHJl1uNt90zUuAJuzqjH4cw6Y=
|
||||
github.com/redis/rueidis v1.0.64/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.64 h1:M8JbLP4LyHQhBLBRsUQIzui8/LyTtdESNIMVveqm4RY=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.64/go.mod h1:8pJVPhEjpw0izZFSxYwDziUiEYEkEklTSw/nZzga61M=
|
||||
github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
|
||||
github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM=
|
||||
github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
|
||||
@@ -1620,8 +1620,8 @@ github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138 h1:bX
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
|
||||
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
|
||||
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
|
||||
github.com/seaweedfs/raft v1.1.6 h1:e83Xn0boscPnuSiBllUPeWRVS6JKrqJPYBozgFyBiC0=
|
||||
github.com/seaweedfs/raft v1.1.6/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3MxjuNs=
|
||||
github.com/seaweedfs/raft v1.1.3 h1:5B6hgneQ7IuU4Ceom/f6QUt8pEeqjcsRo+IxlyPZCws=
|
||||
github.com/seaweedfs/raft v1.1.3/go.mod h1:9cYlEBA+djJbnf/5tWsCybtbL7ICYpi+Uxcg3MxjuNs=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
@@ -1779,12 +1779,12 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mo
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 h1:/NyPd9KnCJgzrEXCArqk1ThqCH2Dh31uUwl88o/VkuM=
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0 h1:uvqheUfoEJz0CLLLaYct69wpvoaosM/2joWFcbwoqfw=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0 h1:Ml4WMsaRY+/+Wh3DCp1A+/zjM1aMdQg3irjuYFwWVHQ=
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0/go.mod h1:/LjMxb/rXmoGAAnImoqAFIlhO5ampHacbvDetQitCk4=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1 h1:qw3Fa+T81+Kpu5Io2vYHJOwcrYrVjgJlT6t/0dOXJrA=
|
||||
github.com/ydb-platform/ydb-go-yc v0.12.1/go.mod h1:t/ZA4ECdgPWjAb4jyDe8AzQZB5dhpGbi3iCahFaNwBY=
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 h1:9E5q8Nsy2RiJMZDNVy0A3KUrIMBPakJ2VgloeWbcI84=
|
||||
@@ -1874,8 +1874,8 @@ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A
|
||||
go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
|
||||
go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
|
||||
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
|
||||
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
@@ -1919,8 +1919,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
|
||||
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
|
||||
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
|
||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -1951,8 +1951,8 @@ golang.org/x/image v0.0.0-20210607152325-775e3b0c77b9/go.mod h1:023OzeP/+EPmXeap
|
||||
golang.org/x/image v0.0.0-20210628002857-a66eb6448b8d/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20211028202545-6944b10bf410/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.0.0-20220302094943-723b81ca9867/go.mod h1:023OzeP/+EPmXeapQh35lcL3II3LrY8Ic+EFFKVhULM=
|
||||
golang.org/x/image v0.34.0 h1:33gCkyw9hmwbZJeZkct8XyR11yH889EQt/QH4VmXMn8=
|
||||
golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
|
||||
golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
|
||||
golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -1987,8 +1987,8 @@ golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
|
||||
golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
|
||||
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
|
||||
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -2058,8 +2058,8 @@ golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -2220,8 +2220,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -2239,8 +2239,8 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
|
||||
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
|
||||
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
|
||||
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
|
||||
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -2262,8 +2262,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@@ -2343,8 +2343,8 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58
|
||||
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
|
||||
golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
|
||||
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
|
||||
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
|
||||
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.03"
|
||||
appVersion: "4.02"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.0.403
|
||||
version: 4.0.402
|
||||
|
||||
@@ -1595,101 +1595,6 @@
|
||||
"title": "S3 Bucket Traffic Sent",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "reqps",
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 41
|
||||
},
|
||||
"id": 86,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "S3 API Calls per Bucket",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
@@ -1754,8 +1659,8 @@
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 41
|
||||
},
|
||||
"id": 72,
|
||||
@@ -3361,209 +3266,6 @@
|
||||
],
|
||||
"title": "Filer Go Routines",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 48
|
||||
},
|
||||
"id": 89,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}} (logical)",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}} (physical)",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "S3 Bucket Size",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 48
|
||||
},
|
||||
"id": 90,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "S3 Bucket Object Count",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "",
|
||||
@@ -3654,4 +3356,4 @@
|
||||
"uid": "a24009d7-cbda-4443-a132-1cc1c4677304",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,9 +203,6 @@ spec:
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
- name: admin-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/admin/
|
||||
{{- end }}
|
||||
{{ tpl .Values.admin.extraVolumeMounts . | nindent 12 | trim }}
|
||||
ports:
|
||||
@@ -301,9 +298,6 @@ spec:
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-client-cert
|
||||
- name: admin-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-admin-cert
|
||||
{{- end }}
|
||||
{{ tpl .Values.admin.extraVolumes . | indent 8 | trim }}
|
||||
{{- if .Values.admin.nodeSelector }}
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-admin-cert
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
{{- if .Values.admin.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.admin.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
secretName: {{ template "seaweedfs.name" . }}-admin-cert
|
||||
issuerRef:
|
||||
name: {{ template "seaweedfs.name" . }}-ca-issuer
|
||||
kind: Issuer
|
||||
commonName: {{ .Values.certificates.commonName }}
|
||||
subject:
|
||||
organizations:
|
||||
- "SeaweedFS CA"
|
||||
dnsNames:
|
||||
- '*.{{ template "seaweedfs.name" . }}-admin'
|
||||
- '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}'
|
||||
- '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}.svc'
|
||||
- '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}.svc.cluster.local'
|
||||
{{- if .Values.certificates.ipAddresses }}
|
||||
ipAddresses:
|
||||
{{- range .Values.certificates.ipAddresses }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
privateKey:
|
||||
algorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
size: {{ .Values.certificates.keySize }}
|
||||
duration: {{ .Values.certificates.duration }}
|
||||
renewBefore: {{ .Values.certificates.renewBefore }}
|
||||
{{- end }}
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-worker-cert
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: worker
|
||||
{{- if .Values.worker.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.worker.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
secretName: {{ template "seaweedfs.name" . }}-worker-cert
|
||||
issuerRef:
|
||||
name: {{ template "seaweedfs.name" . }}-ca-issuer
|
||||
kind: Issuer
|
||||
commonName: {{ .Values.certificates.commonName }}
|
||||
subject:
|
||||
organizations:
|
||||
- "SeaweedFS CA"
|
||||
dnsNames:
|
||||
- '*.{{ template "seaweedfs.name" . }}-worker'
|
||||
- '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}'
|
||||
- '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}.svc'
|
||||
- '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}.svc.cluster.local'
|
||||
{{- if .Values.certificates.ipAddresses }}
|
||||
ipAddresses:
|
||||
{{- range .Values.certificates.ipAddresses }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
privateKey:
|
||||
algorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
size: {{ .Values.certificates.keySize }}
|
||||
duration: {{ .Values.certificates.duration }}
|
||||
renewBefore: {{ .Values.certificates.renewBefore }}
|
||||
{{- end }}
|
||||
|
||||
@@ -65,14 +65,6 @@ data:
|
||||
cert = "/usr/local/share/ca-certificates/filer/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/filer/tls.key"
|
||||
|
||||
[grpc.admin]
|
||||
cert = "/usr/local/share/ca-certificates/admin/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/admin/tls.key"
|
||||
|
||||
[grpc.worker]
|
||||
cert = "/usr/local/share/ca-certificates/worker/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/worker/tls.key"
|
||||
|
||||
# use this for any place needs a grpc client
|
||||
# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
|
||||
[grpc.client]
|
||||
|
||||
@@ -171,9 +171,6 @@ spec:
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
- name: worker-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/worker/
|
||||
{{- end }}
|
||||
{{ tpl .Values.worker.extraVolumeMounts . | nindent 12 | trim }}
|
||||
ports:
|
||||
@@ -265,9 +262,6 @@ spec:
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-client-cert
|
||||
- name: worker-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-worker-cert
|
||||
{{- end }}
|
||||
{{ tpl .Values.worker.extraVolumes . | indent 8 | trim }}
|
||||
{{- if .Values.worker.nodeSelector }}
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
# SeaweedFS S3 Java SDK Compatibility Tests
|
||||
|
||||
This project contains Java-based integration tests for SeaweedFS S3 API compatibility.
|
||||
|
||||
## Overview
|
||||
|
||||
Tests are provided for both AWS SDK v1 and v2 to ensure compatibility with the various SDK versions commonly used in production.
|
||||
|
||||
## SDK Versions
|
||||
|
||||
| SDK | Version | Notes |
|
||||
|-----|---------|-------|
|
||||
| AWS SDK v1 for Java | 1.12.600 | Legacy SDK, less strict ETag validation |
|
||||
| AWS SDK v2 for Java | 2.20.127 | Modern SDK with strict checksum validation |
|
||||
|
||||
## Running Tests
|
||||
|
||||
### Prerequisites
|
||||
|
||||
1. SeaweedFS running with S3 API enabled:
|
||||
```bash
|
||||
weed server -s3
|
||||
```
|
||||
|
||||
2. Java 18+ and Maven
|
||||
|
||||
### Run All Tests
|
||||
|
||||
```bash
|
||||
mvn test
|
||||
```
|
||||
|
||||
### Run Specific Tests
|
||||
|
||||
```bash
|
||||
# Run only ETag validation tests (AWS SDK v2)
|
||||
mvn test -Dtest=ETagValidationTest
|
||||
|
||||
# Run with custom endpoint
|
||||
mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `S3_ENDPOINT` | `http://127.0.0.1:8333` | S3 API endpoint URL |
|
||||
| `S3_ACCESS_KEY` | `some_access_key1` | Access key ID |
|
||||
| `S3_SECRET_KEY` | `some_secret_key1` | Secret access key |
|
||||
| `S3_REGION` | `us-east-1` | AWS region |
|
||||
|
||||
## Test Coverage
|
||||
|
||||
### ETagValidationTest (AWS SDK v2)
|
||||
|
||||
Tests for [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768) - ETag format validation.
|
||||
|
||||
| Test | Description |
|
||||
|------|-------------|
|
||||
| `testSmallFilePutObject` | Verify small files return pure MD5 ETag |
|
||||
| `testLargeFilePutObject_Issue7768` | **Critical**: Verify large files (>8MB) return pure MD5 ETag |
|
||||
| `testExtraLargeFilePutObject` | Verify very large files (>24MB) return pure MD5 ETag |
|
||||
| `testMultipartUploadETag` | Verify multipart uploads return composite ETag |
|
||||
| `testETagConsistency` | Verify ETag consistency across PUT/HEAD/GET |
|
||||
| `testMultipleLargeFileUploads` | Stress test multiple large uploads |
|
||||
|
||||
### Background: Issue #7768
|
||||
|
||||
AWS SDK v2 for Java includes checksum validation that decodes the ETag as hexadecimal. When SeaweedFS returned composite ETags (`<md5>-<count>`) for regular `PutObject` with internally auto-chunked files, the SDK failed with:
|
||||
|
||||
```
|
||||
java.lang.IllegalArgumentException: Invalid base 16 character: '-'
|
||||
```
|
||||
|
||||
**Per AWS S3 specification:**
|
||||
- `PutObject`: ETag is always a pure MD5 hex string (32 chars)
|
||||
- `CompleteMultipartUpload`: ETag is composite format (`<md5>-<partcount>`)
|
||||
|
||||
The fix ensures SeaweedFS follows this specification.
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
src/
|
||||
├── main/java/com/seaweedfs/s3/
|
||||
│ ├── PutObject.java # Example PutObject with SDK v1
|
||||
│ └── HighLevelMultipartUpload.java
|
||||
└── test/java/com/seaweedfs/s3/
|
||||
├── PutObjectTest.java # Basic SDK v1 test
|
||||
└── ETagValidationTest.java # Comprehensive SDK v2 ETag tests
|
||||
```
|
||||
|
||||
## Validated SDK Versions
|
||||
|
||||
This Java test project validates:
|
||||
|
||||
- ✅ AWS SDK v2 for Java 2.20.127+
|
||||
- ✅ AWS SDK v1 for Java 1.12.600+
|
||||
|
||||
Go SDK validation is performed by separate test suites:
|
||||
- See [Go ETag Tests](/test/s3/etag/) for AWS SDK v2 for Go tests
|
||||
- See [test/s3/SDK_COMPATIBILITY.md](/test/s3/SDK_COMPATIBILITY.md) for full SDK compatibility matrix
|
||||
|
||||
## Related
|
||||
|
||||
- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
|
||||
- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
|
||||
- [Go ETag Tests](/test/s3/etag/)
|
||||
- [SDK Compatibility Matrix](/test/s3/SDK_COMPATIBILITY.md)
|
||||
|
||||
@@ -6,29 +6,18 @@
|
||||
<packaging>jar</packaging>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<aws.sdk.v1.version>1.12.600</aws.sdk.v1.version>
|
||||
<aws.sdk.v2.version>2.20.127</aws.sdk.v2.version>
|
||||
<maven.compiler.source>18</maven.compiler.source>
|
||||
<maven.compiler.target>18</maven.compiler.target>
|
||||
</properties>
|
||||
<name>copier</name>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<!-- AWS SDK v1 BOM -->
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-bom</artifactId>
|
||||
<version>${aws.sdk.v1.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
<!-- AWS SDK v2 BOM -->
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>bom</artifactId>
|
||||
<version>${aws.sdk.v2.version}</version>
|
||||
<version>1.11.327</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
@@ -36,50 +25,15 @@
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<!-- AWS SDK v1 (for backward compatibility with existing code) -->
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-s3</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- AWS SDK v2 (for modern tests with checksum validation) -->
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>s3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>s3-transfer-manager</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Test dependencies -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.13.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>5.10.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<version>3.24.2</version>
|
||||
<version>4.13.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>3.2.2</version>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
@@ -1,439 +0,0 @@
|
||||
package com.seaweedfs.s3;
|
||||
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.*;
|
||||
|
||||
import java.net.URI;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.assertj.core.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* AWS SDK v2 Integration Tests for S3 ETag Format Validation.
|
||||
*
|
||||
* These tests verify that SeaweedFS returns correct ETag formats that are
|
||||
* compatible with AWS SDK v2's checksum validation.
|
||||
*
|
||||
* Background (GitHub Issue #7768):
|
||||
* AWS SDK v2 for Java validates ETags as hexadecimal MD5 hashes for PutObject
|
||||
* responses. If the ETag contains non-hex characters (like '-' in composite
|
||||
* format), the SDK fails with "Invalid base 16 character: '-'".
|
||||
*
|
||||
* Per AWS S3 specification:
|
||||
* - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
|
||||
* - CompleteMultipartUpload: ETag is composite format "<md5>-<partcount>"
|
||||
*
|
||||
* To run these tests:
|
||||
* mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
|
||||
*
|
||||
* Or set environment variable:
|
||||
* export S3_ENDPOINT=http://localhost:8333
|
||||
* mvn test -Dtest=ETagValidationTest
|
||||
*/
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
@DisplayName("S3 ETag Format Validation Tests (AWS SDK v2)")
|
||||
class ETagValidationTest {
|
||||
|
||||
// Configuration - can be overridden via system properties or environment variables
|
||||
private static final String DEFAULT_ENDPOINT = "http://127.0.0.1:8333";
|
||||
private static final String DEFAULT_ACCESS_KEY = "some_access_key1";
|
||||
private static final String DEFAULT_SECRET_KEY = "some_secret_key1";
|
||||
private static final String DEFAULT_REGION = "us-east-1";
|
||||
|
||||
// Auto-chunking threshold in SeaweedFS (must match s3api_object_handlers_put.go)
|
||||
private static final int AUTO_CHUNK_SIZE = 8 * 1024 * 1024; // 8MB
|
||||
|
||||
// Test sizes
|
||||
private static final int SMALL_FILE_SIZE = 1024; // 1KB
|
||||
private static final int LARGE_FILE_SIZE = 10 * 1024 * 1024; // 10MB (triggers auto-chunking)
|
||||
private static final int XL_FILE_SIZE = 25 * 1024 * 1024; // 25MB (multiple chunks)
|
||||
private static final int MULTIPART_PART_SIZE = 5 * 1024 * 1024; // 5MB per part
|
||||
|
||||
// ETag format patterns
|
||||
private static final Pattern PURE_MD5_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}\"?$");
|
||||
private static final Pattern COMPOSITE_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}-\\d+\"?$");
|
||||
|
||||
private S3Client s3Client;
|
||||
private String testBucketName;
|
||||
private final SecureRandom random = new SecureRandom();
|
||||
|
||||
@BeforeAll
|
||||
void setUp() {
|
||||
String endpoint = getConfig("S3_ENDPOINT", DEFAULT_ENDPOINT);
|
||||
String accessKey = getConfig("S3_ACCESS_KEY", DEFAULT_ACCESS_KEY);
|
||||
String secretKey = getConfig("S3_SECRET_KEY", DEFAULT_SECRET_KEY);
|
||||
String region = getConfig("S3_REGION", DEFAULT_REGION);
|
||||
|
||||
System.out.println("Connecting to S3 endpoint: " + endpoint);
|
||||
|
||||
s3Client = S3Client.builder()
|
||||
.endpointOverride(URI.create(endpoint))
|
||||
.credentialsProvider(StaticCredentialsProvider.create(
|
||||
AwsBasicCredentials.create(accessKey, secretKey)))
|
||||
.region(Region.of(region))
|
||||
.forcePathStyle(true) // Required for SeaweedFS
|
||||
.build();
|
||||
|
||||
// Create test bucket
|
||||
testBucketName = "test-etag-" + UUID.randomUUID().toString().substring(0, 8);
|
||||
s3Client.createBucket(CreateBucketRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.build());
|
||||
|
||||
System.out.println("Created test bucket: " + testBucketName);
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
void tearDown() {
|
||||
if (s3Client != null && testBucketName != null) {
|
||||
try {
|
||||
// Delete all objects with pagination
|
||||
String continuationToken = null;
|
||||
do {
|
||||
ListObjectsV2Response listResp = s3Client.listObjectsV2(
|
||||
ListObjectsV2Request.builder()
|
||||
.bucket(testBucketName)
|
||||
.continuationToken(continuationToken)
|
||||
.build());
|
||||
for (S3Object obj : listResp.contents()) {
|
||||
s3Client.deleteObject(DeleteObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(obj.key())
|
||||
.build());
|
||||
}
|
||||
continuationToken = listResp.nextContinuationToken();
|
||||
} while (continuationToken != null);
|
||||
|
||||
// Abort any multipart uploads
|
||||
ListMultipartUploadsResponse mpResp = s3Client.listMultipartUploads(
|
||||
ListMultipartUploadsRequest.builder().bucket(testBucketName).build());
|
||||
for (MultipartUpload upload : mpResp.uploads()) {
|
||||
s3Client.abortMultipartUpload(AbortMultipartUploadRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(upload.key())
|
||||
.uploadId(upload.uploadId())
|
||||
.build());
|
||||
}
|
||||
|
||||
// Delete bucket
|
||||
s3Client.deleteBucket(DeleteBucketRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.build());
|
||||
|
||||
System.out.println("Cleaned up test bucket: " + testBucketName);
|
||||
} catch (Exception e) {
|
||||
System.err.println("Warning: Failed to cleanup test bucket: " + e.getMessage());
|
||||
}
|
||||
s3Client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Small file PutObject should return pure MD5 ETag")
|
||||
void testSmallFilePutObject() throws Exception {
|
||||
byte[] testData = generateRandomData(SMALL_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "small-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
System.out.println("Small file ETag: " + etag + " (expected MD5: " + expectedMD5 + ")");
|
||||
|
||||
assertThat(etag)
|
||||
.describedAs("Small file ETag should be pure MD5")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("ETag should match calculated MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
assertThat(etag)
|
||||
.describedAs("ETag should not contain hyphen")
|
||||
.doesNotContain("-");
|
||||
}
|
||||
|
||||
/**
|
||||
* Critical test for GitHub Issue #7768.
|
||||
*
|
||||
* This test uploads a file larger than the auto-chunking threshold (8MB),
|
||||
* which triggers SeaweedFS to split the file into multiple internal chunks.
|
||||
*
|
||||
* Previously, this caused SeaweedFS to return a composite ETag like
|
||||
* "d41d8cd98f00b204e9800998ecf8427e-2", which AWS SDK v2 rejected because
|
||||
* it validates the ETag as hexadecimal and '-' is not a valid hex character.
|
||||
*
|
||||
* The fix ensures that regular PutObject always returns a pure MD5 ETag,
|
||||
* regardless of internal chunking.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Large file PutObject (>8MB) should return pure MD5 ETag - Issue #7768")
|
||||
void testLargeFilePutObject_Issue7768() throws Exception {
|
||||
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "large-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
System.out.println("Uploading large file (" + LARGE_FILE_SIZE + " bytes, " +
|
||||
"> " + AUTO_CHUNK_SIZE + " byte auto-chunk threshold)...");
|
||||
|
||||
// This is where Issue #7768 would manifest - SDK v2 validates ETag
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
int expectedChunks = (LARGE_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
|
||||
System.out.println("Large file ETag: " + etag +
|
||||
" (expected MD5: " + expectedMD5 + ", internal chunks: ~" + expectedChunks + ")");
|
||||
|
||||
// These assertions would fail before the fix
|
||||
assertThat(etag)
|
||||
.describedAs("Large file PutObject ETag MUST be pure MD5 (not composite)")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(etag)
|
||||
.describedAs("Large file ETag should NOT be composite format")
|
||||
.doesNotMatch(COMPOSITE_PATTERN);
|
||||
assertThat(etag)
|
||||
.describedAs("ETag should not contain hyphen for regular PutObject")
|
||||
.doesNotContain("-");
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("ETag should match calculated MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
|
||||
// Verify hex decoding works (this is what fails in Issue #7768)
|
||||
assertThatCode(() -> hexToBytes(cleanETag(etag)))
|
||||
.describedAs("ETag should be valid hexadecimal (AWS SDK v2 validation)")
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Extra large file PutObject (>24MB) should return pure MD5 ETag")
|
||||
void testExtraLargeFilePutObject() throws Exception {
|
||||
byte[] testData = generateRandomData(XL_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "xl-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
int expectedChunks = (XL_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
|
||||
System.out.println("Uploading XL file (" + XL_FILE_SIZE + " bytes, ~" +
|
||||
expectedChunks + " internal chunks)...");
|
||||
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
System.out.println("XL file ETag: " + etag);
|
||||
|
||||
assertThat(etag)
|
||||
.describedAs("XL file PutObject ETag MUST be pure MD5")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("ETag should match calculated MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Multipart upload should return composite ETag")
|
||||
void testMultipartUploadETag() throws Exception {
|
||||
int totalSize = 15 * 1024 * 1024; // 15MB = 3 parts
|
||||
byte[] testData = generateRandomData(totalSize);
|
||||
String objectKey = "multipart-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
System.out.println("Performing multipart upload (" + totalSize + " bytes)...");
|
||||
|
||||
// Initiate multipart upload
|
||||
CreateMultipartUploadResponse createResp = s3Client.createMultipartUpload(
|
||||
CreateMultipartUploadRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build());
|
||||
String uploadId = createResp.uploadId();
|
||||
|
||||
List<CompletedPart> completedParts = new ArrayList<>();
|
||||
int partNumber = 1;
|
||||
|
||||
// Upload parts
|
||||
for (int offset = 0; offset < totalSize; offset += MULTIPART_PART_SIZE) {
|
||||
int end = Math.min(offset + MULTIPART_PART_SIZE, totalSize);
|
||||
byte[] partData = new byte[end - offset];
|
||||
System.arraycopy(testData, offset, partData, 0, partData.length);
|
||||
|
||||
UploadPartResponse uploadResp = s3Client.uploadPart(
|
||||
UploadPartRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.uploadId(uploadId)
|
||||
.partNumber(partNumber)
|
||||
.build(),
|
||||
RequestBody.fromBytes(partData));
|
||||
|
||||
completedParts.add(CompletedPart.builder()
|
||||
.partNumber(partNumber)
|
||||
.eTag(uploadResp.eTag())
|
||||
.build());
|
||||
partNumber++;
|
||||
}
|
||||
|
||||
// Complete multipart upload
|
||||
CompleteMultipartUploadResponse completeResp = s3Client.completeMultipartUpload(
|
||||
CompleteMultipartUploadRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.uploadId(uploadId)
|
||||
.multipartUpload(CompletedMultipartUpload.builder()
|
||||
.parts(completedParts)
|
||||
.build())
|
||||
.build());
|
||||
|
||||
String etag = completeResp.eTag();
|
||||
System.out.println("Multipart upload ETag: " + etag + " (" + completedParts.size() + " parts)");
|
||||
|
||||
// Multipart uploads SHOULD have composite ETag
|
||||
assertThat(etag)
|
||||
.describedAs("Multipart upload ETag SHOULD be composite format")
|
||||
.matches(COMPOSITE_PATTERN);
|
||||
assertThat(etag)
|
||||
.describedAs("Multipart ETag should contain hyphen")
|
||||
.contains("-");
|
||||
|
||||
// Verify part count in ETag
|
||||
String[] parts = cleanETag(etag).split("-");
|
||||
assertThat(parts).hasSize(2);
|
||||
assertThat(parts[1])
|
||||
.describedAs("Part count in ETag should match uploaded parts")
|
||||
.isEqualTo(String.valueOf(completedParts.size()));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("ETag should be consistent across PUT, HEAD, and GET")
|
||||
void testETagConsistency() throws Exception {
|
||||
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
|
||||
String objectKey = "consistency-test-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
// PUT
|
||||
PutObjectResponse putResp = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
String putETag = putResp.eTag();
|
||||
|
||||
// HEAD
|
||||
HeadObjectResponse headResp = s3Client.headObject(
|
||||
HeadObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build());
|
||||
String headETag = headResp.eTag();
|
||||
|
||||
// GET
|
||||
GetObjectResponse getResp = s3Client.getObject(
|
||||
GetObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build())
|
||||
.response();
|
||||
String getETag = getResp.eTag();
|
||||
|
||||
System.out.println("PUT ETag: " + putETag + ", HEAD ETag: " + headETag + ", GET ETag: " + getETag);
|
||||
|
||||
assertThat(putETag).isEqualTo(headETag);
|
||||
assertThat(putETag).isEqualTo(getETag);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Multiple large file uploads should all return pure MD5 ETags")
|
||||
void testMultipleLargeFileUploads() throws Exception {
|
||||
int numFiles = 3;
|
||||
|
||||
for (int i = 0; i < numFiles; i++) {
|
||||
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "multi-large-" + i + "-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
System.out.println("File " + i + " ETag: " + etag);
|
||||
|
||||
assertThat(etag)
|
||||
.describedAs("File " + i + " ETag should be pure MD5")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("File " + i + " ETag should match MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
|
||||
// Validate hex decoding
|
||||
assertThatCode(() -> hexToBytes(cleanETag(etag)))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
}
|
||||
|
||||
// Helper methods
|
||||
|
||||
private String getConfig(String key, String defaultValue) {
|
||||
String value = System.getProperty(key);
|
||||
if (value == null) {
|
||||
value = System.getenv(key);
|
||||
}
|
||||
return value != null ? value : defaultValue;
|
||||
}
|
||||
|
||||
private byte[] generateRandomData(int size) {
|
||||
byte[] data = new byte[size];
|
||||
random.nextBytes(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
private String calculateMD5Hex(byte[] data) throws Exception {
|
||||
MessageDigest md = MessageDigest.getInstance("MD5");
|
||||
byte[] digest = md.digest(data);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private String cleanETag(String etag) {
|
||||
if (etag == null) return null;
|
||||
return etag.replace("\"", "");
|
||||
}
|
||||
|
||||
private byte[] hexToBytes(String hex) {
|
||||
int len = hex.length();
|
||||
byte[] data = new byte[len / 2];
|
||||
for (int i = 0; i < len; i += 2) {
|
||||
data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4)
|
||||
+ Character.digit(hex.charAt(i + 1), 16));
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1319,99 +1319,6 @@
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 90,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 API Calls per Bucket",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "reqps",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
@@ -1425,8 +1332,8 @@
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
@@ -3053,200 +2960,6 @@
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 37
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 87,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}} (logical)",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}} (physical)",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 Bucket Size",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "decbytes",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 37
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 88,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 Bucket Object Count",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
|
||||
@@ -875,85 +875,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 86,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "S3 API Calls per Bucket",
|
||||
"tooltip": {
|
||||
"msResolution": true,
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "cumulative"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "reqps",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
@@ -1166,202 +1087,6 @@
|
||||
"show": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 37
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 87,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"span": 6,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}} (logical)",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}} (physical)",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 Bucket Size",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "decbytes",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 37
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 88,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"span": 6,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 Bucket Object Count",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"repeat": null,
|
||||
@@ -2386,4 +2111,4 @@
|
||||
"timezone": "browser",
|
||||
"title": "SeaweedFS ks8 heartbeat",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
|
||||
@@ -447,99 +447,6 @@
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 86,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$namespace\",service=~\"$service-api\"}[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 API Calls per Bucket",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "reqps",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
@@ -559,8 +466,8 @@
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
@@ -670,8 +577,8 @@
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
@@ -781,8 +688,8 @@
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
@@ -2447,209 +2354,6 @@
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 41
|
||||
},
|
||||
"id": 89,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}} (logical)",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}} (physical)",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "S3 Bucket Size",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 41
|
||||
},
|
||||
"id": 90,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "S3 Bucket Object Count",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
@@ -2841,4 +2545,4 @@
|
||||
"timezone": "browser",
|
||||
"title": "SeaweedFS",
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,187 +0,0 @@
|
||||
# Makefile for EC integration testing
|
||||
# Usage:
|
||||
# make start - Start the test cluster (master + 6 volume servers + filer)
|
||||
# make stop - Stop the test cluster
|
||||
# make populate - Populate test data (~300MB across 7 volumes)
|
||||
# make shell - Open weed shell connected to the test cluster
|
||||
# make clean - Stop cluster and remove all test data
|
||||
# make setup - Start cluster and populate data (one command)
|
||||
#
|
||||
# Requirements: curl, jq
|
||||
|
||||
WEED_BINARY := $(shell pwd)/../../weed/weed
|
||||
TEST_DIR := /tmp/ec_manual_test
|
||||
# Use non-standard ports to avoid conflicts with existing SeaweedFS servers
|
||||
MASTER_PORT := 29333
|
||||
FILER_PORT := 28888
|
||||
VOLUME_BASE_PORT := 28080
|
||||
NUM_VOLUME_SERVERS := 6
|
||||
VOLUME_SIZE_LIMIT_MB := 30
|
||||
MAX_VOLUMES_PER_SERVER := 10
|
||||
|
||||
# Build weed binary if it doesn't exist
|
||||
$(WEED_BINARY):
|
||||
cd ../../weed && go build -o weed .
|
||||
|
||||
.PHONY: build
|
||||
build: $(WEED_BINARY)
|
||||
|
||||
.PHONY: start
|
||||
start: build
|
||||
@echo "=== Starting SeaweedFS test cluster ==="
|
||||
@mkdir -p $(TEST_DIR)/master $(TEST_DIR)/filer
|
||||
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do mkdir -p $(TEST_DIR)/volume$$i; done
|
||||
@# Create security.toml with JWT disabled
|
||||
@echo "# Disable JWT for testing" > $(TEST_DIR)/security.toml
|
||||
@echo '[jwt.signing]' >> $(TEST_DIR)/security.toml
|
||||
@echo 'key = ""' >> $(TEST_DIR)/security.toml
|
||||
@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
|
||||
@echo '' >> $(TEST_DIR)/security.toml
|
||||
@echo '[jwt.signing.read]' >> $(TEST_DIR)/security.toml
|
||||
@echo 'key = ""' >> $(TEST_DIR)/security.toml
|
||||
@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
|
||||
@# Create filer.toml with leveldb2
|
||||
@echo '[leveldb2]' > $(TEST_DIR)/filer.toml
|
||||
@echo 'enabled = true' >> $(TEST_DIR)/filer.toml
|
||||
@echo 'dir = "$(TEST_DIR)/filer/filerldb2"' >> $(TEST_DIR)/filer.toml
|
||||
@# Start master
|
||||
@echo "Starting master on port $(MASTER_PORT)..."
|
||||
@cd $(TEST_DIR) && $(WEED_BINARY) master \
|
||||
-port=$(MASTER_PORT) \
|
||||
-mdir=$(TEST_DIR)/master \
|
||||
-volumeSizeLimitMB=$(VOLUME_SIZE_LIMIT_MB) \
|
||||
-ip=127.0.0.1 \
|
||||
> $(TEST_DIR)/master/master.log 2>&1 & echo $$! > $(TEST_DIR)/master.pid
|
||||
@sleep 3
|
||||
@# Start volume servers (run from TEST_DIR to find security.toml)
|
||||
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
|
||||
port=$$(($(VOLUME_BASE_PORT) + $$i)); \
|
||||
echo "Starting volume server $$i on port $$port (rack$$i)..."; \
|
||||
cd $(TEST_DIR) && $(WEED_BINARY) volume \
|
||||
-port=$$port \
|
||||
-dir=$(TEST_DIR)/volume$$i \
|
||||
-max=$(MAX_VOLUMES_PER_SERVER) \
|
||||
-master=127.0.0.1:$(MASTER_PORT) \
|
||||
-ip=127.0.0.1 \
|
||||
-dataCenter=dc1 \
|
||||
-rack=rack$$i \
|
||||
> $(TEST_DIR)/volume$$i/volume.log 2>&1 & echo $$! > $(TEST_DIR)/volume$$i.pid; \
|
||||
done
|
||||
@sleep 3
|
||||
@# Start filer (run from TEST_DIR to find security.toml)
|
||||
@echo "Starting filer on port $(FILER_PORT)..."
|
||||
@cd $(TEST_DIR) && $(WEED_BINARY) filer \
|
||||
-port=$(FILER_PORT) \
|
||||
-master=127.0.0.1:$(MASTER_PORT) \
|
||||
-ip=127.0.0.1 \
|
||||
> $(TEST_DIR)/filer/filer.log 2>&1 & echo $$! > $(TEST_DIR)/filer.pid
|
||||
@sleep 3
|
||||
@echo ""
|
||||
@echo "=== Cluster started ==="
|
||||
@echo "Master: http://127.0.0.1:$(MASTER_PORT)"
|
||||
@echo "Filer: http://127.0.0.1:$(FILER_PORT)"
|
||||
@echo "Volume servers: http://127.0.0.1:$(VOLUME_BASE_PORT) - http://127.0.0.1:$$(($(VOLUME_BASE_PORT) + $(NUM_VOLUME_SERVERS) - 1))"
|
||||
@echo ""
|
||||
@echo "Run 'make shell' to open weed shell"
|
||||
@echo "Run 'make populate' to add test data"
|
||||
|
||||
.PHONY: stop
|
||||
stop:
|
||||
@echo "=== Stopping SeaweedFS test cluster ==="
|
||||
@# Stop filer by PID
|
||||
@-[ -f $(TEST_DIR)/filer.pid ] && kill $$(cat $(TEST_DIR)/filer.pid) 2>/dev/null && rm -f $(TEST_DIR)/filer.pid || true
|
||||
@# Stop volume servers by PID
|
||||
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
|
||||
[ -f $(TEST_DIR)/volume$$i.pid ] && kill $$(cat $(TEST_DIR)/volume$$i.pid) 2>/dev/null && rm -f $(TEST_DIR)/volume$$i.pid || true; \
|
||||
done
|
||||
@# Stop master by PID
|
||||
@-[ -f $(TEST_DIR)/master.pid ] && kill $$(cat $(TEST_DIR)/master.pid) 2>/dev/null && rm -f $(TEST_DIR)/master.pid || true
|
||||
@# Fallback: use pkill with specific patterns to ensure cleanup
|
||||
@-pkill -f "weed filer.*-master=127.0.0.1:$(MASTER_PORT)" 2>/dev/null || true
|
||||
@-pkill -f "weed volume.*-dir=$(TEST_DIR)/volume" 2>/dev/null || true
|
||||
@-pkill -f "weed master.*-mdir=$(TEST_DIR)/master" 2>/dev/null || true
|
||||
@echo "Cluster stopped."
|
||||
|
||||
.PHONY: clean
|
||||
clean: stop
|
||||
@echo "Removing test data..."
|
||||
@rm -rf $(TEST_DIR)
|
||||
@echo "Clean complete."
|
||||
|
||||
.PHONY: populate
|
||||
populate:
|
||||
@echo "=== Populating test data (~300MB) ==="
|
||||
@# Create a 500KB test file template using mktemp for isolation
|
||||
@tmpfile=$$(mktemp) && \
|
||||
dd if=/dev/urandom bs=1024 count=500 of=$$tmpfile 2>/dev/null && \
|
||||
uploaded=0; \
|
||||
for i in $$(seq 1 600); do \
|
||||
response=$$(curl -s "http://127.0.0.1:$(MASTER_PORT)/dir/assign?collection=ectest&replication=000"); \
|
||||
fid=$$(echo $$response | jq -r '.fid'); \
|
||||
url=$$(echo $$response | jq -r '.url'); \
|
||||
if [ "$$fid" != "null" ] && [ -n "$$fid" ]; then \
|
||||
curl -s -F "file=@$$tmpfile;filename=file_$$i.bin" "http://$$url/$$fid" > /dev/null; \
|
||||
uploaded=$$((uploaded + 1)); \
|
||||
fi; \
|
||||
if [ $$((i % 100)) -eq 0 ]; then \
|
||||
echo "Uploaded $$uploaded files..."; \
|
||||
fi; \
|
||||
done; \
|
||||
rm -f $$tmpfile; \
|
||||
echo ""; \
|
||||
echo "=== Data population complete ==="; \
|
||||
echo "Uploaded $$uploaded files (~$$((uploaded * 500 / 1024))MB)"
|
||||
@echo ""
|
||||
@echo "Volume status:"
|
||||
@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
|
||||
'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | " Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack)"' 2>/dev/null || true
|
||||
|
||||
.PHONY: shell
|
||||
shell: build
|
||||
@echo "Opening weed shell..."
|
||||
@echo "Commands to try:"
|
||||
@echo " lock"
|
||||
@echo " volume.list"
|
||||
@echo " ec.encode -collection=ectest -quietFor=1s -force"
|
||||
@echo " ec.balance -collection=ectest"
|
||||
@echo " unlock"
|
||||
@echo ""
|
||||
@$(WEED_BINARY) shell -master=127.0.0.1:$(MASTER_PORT) -filer=127.0.0.1:$(FILER_PORT)
|
||||
|
||||
.PHONY: setup
|
||||
setup: clean start
|
||||
@sleep 2
|
||||
@$(MAKE) populate
|
||||
|
||||
.PHONY: status
|
||||
status:
|
||||
@echo "=== Cluster Status ==="
|
||||
@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
|
||||
'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack) (\($$server))"' 2>/dev/null | sort -t: -k1 -n || echo "Cluster not running"
|
||||
@echo ""
|
||||
@echo "=== EC Shards ==="
|
||||
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
|
||||
count=$$(ls $(TEST_DIR)/volume$$i/*.ec[0-9]* 2>/dev/null | wc -l | tr -d ' '); \
|
||||
if [ "$$count" != "0" ]; then \
|
||||
echo " volume$$i (port $$(($(VOLUME_BASE_PORT) + $$i))): $$count EC shard files"; \
|
||||
fi; \
|
||||
done
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@echo "EC Integration Test Makefile"
|
||||
@echo ""
|
||||
@echo "Targets:"
|
||||
@echo " make start - Start test cluster (master + 6 volume servers + filer)"
|
||||
@echo " make stop - Stop test cluster"
|
||||
@echo " make populate - Populate ~300MB of test data"
|
||||
@echo " make shell - Open weed shell"
|
||||
@echo " make setup - Clean, start, and populate (all-in-one)"
|
||||
@echo " make status - Show cluster and EC shard status"
|
||||
@echo " make clean - Stop cluster and remove all test data"
|
||||
@echo " make help - Show this help"
|
||||
@echo ""
|
||||
@echo "Quick start:"
|
||||
@echo " make setup # Start cluster and populate data"
|
||||
@echo " make shell # Open shell to run EC commands"
|
||||
|
||||
@@ -78,43 +78,6 @@ go test -v -run TestECEncodingMasterTimingRaceCondition
|
||||
go test -v -short
|
||||
```
|
||||
|
||||
## Manual Testing with Makefile
|
||||
|
||||
A Makefile is provided for manual EC testing.
|
||||
|
||||
**Requirements:** `curl`, `jq` (command-line JSON processor)
|
||||
|
||||
```bash
|
||||
# Quick start: start cluster and populate data
|
||||
make setup
|
||||
|
||||
# Open weed shell to run EC commands
|
||||
make shell
|
||||
|
||||
# Individual targets
|
||||
make start # Start test cluster (master + 6 volume servers + filer)
|
||||
make stop # Stop test cluster
|
||||
make populate # Populate ~300MB of test data
|
||||
make status # Show cluster and EC shard status
|
||||
make clean # Stop cluster and remove all test data
|
||||
make help # Show all targets
|
||||
```
|
||||
|
||||
### EC Rebalance Limited Slots (Unit Test)
|
||||
|
||||
The "no free ec shard slots" issue is tested with a **unit test** that works directly on
|
||||
topology data structures without requiring a running cluster.
|
||||
|
||||
**Location**: `weed/shell/ec_rebalance_slots_test.go`
|
||||
|
||||
Tests included:
|
||||
- `TestECRebalanceWithLimitedSlots`: Tests a topology with 6 servers, 7 EC volumes (98 shards)
|
||||
- `TestECRebalanceZeroFreeSlots`: Reproduces the exact 0 free slots scenario
|
||||
|
||||
**Known Issue**: When volume servers are at capacity (`volumeCount == maxVolumeCount`),
|
||||
the rebalance step fails with "no free ec shard slots" instead of recognizing that
|
||||
moving shards frees slots on source servers.
|
||||
|
||||
## Test Results
|
||||
|
||||
**With the fix**: Shows "Collecting volume locations for N volumes before EC encoding..." message
|
||||
|
||||
@@ -15,10 +15,8 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/shell"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
@@ -98,12 +96,13 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
|
||||
// This simulates the race condition where EC encoding updates master metadata
|
||||
// but volume location collection happens after that update
|
||||
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// First acquire the lock (required for EC encode)
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err = lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Execute EC encoding - test the timing directly
|
||||
var encodeOutput bytes.Buffer
|
||||
@@ -388,7 +387,7 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
// Find weed binary
|
||||
weedBinary := findWeedBinary()
|
||||
if weedBinary == "" {
|
||||
return nil, fmt.Errorf("weed binary not found - build with 'go build' or 'make' first")
|
||||
return nil, fmt.Errorf("weed binary not found")
|
||||
}
|
||||
|
||||
cluster := &TestCluster{}
|
||||
@@ -397,11 +396,6 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
masterDir := filepath.Join(dataDir, "master")
|
||||
os.MkdirAll(masterDir, 0755)
|
||||
|
||||
// Create an empty security.toml to disable JWT authentication in tests
|
||||
// This prevents the test from picking up ~/.seaweedfs/security.toml
|
||||
securityToml := filepath.Join(dataDir, "security.toml")
|
||||
os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
|
||||
|
||||
// Start master server
|
||||
masterCmd := exec.CommandContext(ctx, weedBinary, "master",
|
||||
"-port", "9333",
|
||||
@@ -410,7 +404,6 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
"-ip", "127.0.0.1",
|
||||
"-peers", "none", // Faster startup when no multiple masters needed
|
||||
)
|
||||
masterCmd.Dir = dataDir // Run from test dir so it picks up our security.toml
|
||||
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
if err != nil {
|
||||
@@ -443,7 +436,6 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
"-dataCenter", "dc1",
|
||||
"-rack", rack,
|
||||
)
|
||||
volumeCmd.Dir = dataDir // Run from test dir so it picks up our security.toml
|
||||
|
||||
volumeLogFile, err := os.Create(filepath.Join(volumeDir, "volume.log"))
|
||||
if err != nil {
|
||||
@@ -478,10 +470,6 @@ func findWeedBinary() string {
|
||||
|
||||
for _, candidate := range candidates {
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
// Convert to absolute path so it works when command's Dir is changed
|
||||
if absPath, err := filepath.Abs(candidate); err == nil {
|
||||
return absPath
|
||||
}
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
@@ -663,55 +651,6 @@ type commandRunner interface {
|
||||
Do([]string, *shell.CommandEnv, io.Writer) error
|
||||
}
|
||||
|
||||
// tryLockWithTimeout attempts to acquire the shell lock with a timeout.
|
||||
// Returns true if lock was acquired, false if timeout or error occurred.
|
||||
// If lock was acquired, the caller must call the returned unlock function.
|
||||
func tryLockWithTimeout(t *testing.T, commandEnv *shell.CommandEnv, timeout time.Duration) (locked bool, unlock func()) {
|
||||
t.Helper()
|
||||
|
||||
type lockResult struct {
|
||||
err error
|
||||
output string
|
||||
}
|
||||
|
||||
lockDone := make(chan lockResult, 1)
|
||||
go func() {
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
lockDone <- lockResult{err: err, output: lockOutput.String()}
|
||||
}()
|
||||
|
||||
select {
|
||||
case res := <-lockDone:
|
||||
if res.err != nil {
|
||||
t.Logf("lock command failed: %v, output: %s", res.err, res.output)
|
||||
return false, nil
|
||||
}
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
return true, func() {
|
||||
var unlockOutput bytes.Buffer
|
||||
_ = unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
}
|
||||
case <-time.After(timeout):
|
||||
t.Logf("timed out acquiring lock after %s", timeout)
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// connectToMasterAndSync connects the command environment to the master and waits for sync.
|
||||
// This helper reduces code duplication across test functions.
|
||||
func connectToMasterAndSync(ctx context.Context, t *testing.T, commandEnv *shell.CommandEnv) {
|
||||
t.Helper()
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
// Wait for master client to fully sync
|
||||
t.Log("Waiting for master client to sync...")
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
|
||||
// captureCommandOutput executes a shell command and captures its output from both
|
||||
// stdout/stderr and the command's buffer. This reduces code duplication in tests.
|
||||
func captureCommandOutput(t *testing.T, cmd commandRunner, args []string, commandEnv *shell.CommandEnv) (output string, err error) {
|
||||
@@ -722,16 +661,17 @@ func captureCommandOutput(t *testing.T, cmd commandRunner, args []string, comman
|
||||
r, w, pipeErr := os.Pipe()
|
||||
require.NoError(t, pipeErr)
|
||||
|
||||
defer func() {
|
||||
_ = w.Close()
|
||||
os.Stdout = oldStdout
|
||||
os.Stderr = oldStderr
|
||||
}()
|
||||
|
||||
os.Stdout = w
|
||||
os.Stderr = w
|
||||
|
||||
cmdErr := cmd.Do(args, commandEnv, &outBuf)
|
||||
|
||||
// Close write end BEFORE reading to signal EOF to the reader
|
||||
_ = w.Close()
|
||||
os.Stdout = oldStdout
|
||||
os.Stderr = oldStderr
|
||||
|
||||
capturedOutput, readErr := io.ReadAll(r)
|
||||
_ = r.Close()
|
||||
require.NoError(t, readErr)
|
||||
@@ -813,7 +753,12 @@ func TestDiskAwareECRebalancing(t *testing.T) {
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
connectToMasterAndSync(ctx, t, commandEnv)
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
// Wait for master client to fully sync
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// Upload test data to create a volume - retry if volumes not ready
|
||||
var volumeId needle.VolumeId
|
||||
@@ -1247,7 +1192,12 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
connectToMasterAndSync(ctx, t, commandEnv)
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
// Wait for master client to fully sync
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// Upload test data to create a volume - retry if volumes not ready
|
||||
var volumeId needle.VolumeId
|
||||
@@ -1274,12 +1224,19 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_encode_with_ssd_disktype", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock first
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Defer unlock to ensure it's always released
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// Execute EC encoding with SSD disk type
|
||||
ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
|
||||
@@ -1309,12 +1266,19 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_balance_with_ssd_disktype", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock first
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Defer unlock to ensure it's always released
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// Execute EC balance with SSD disk type
|
||||
ecBalanceCmd := shell.Commands[findCommandIndex("ec.balance")]
|
||||
@@ -1357,12 +1321,19 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_encode_with_source_disktype", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Test that -sourceDiskType flag is accepted
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Defer unlock to ensure it's always released
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// Execute EC encoding with sourceDiskType filter
|
||||
ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
|
||||
@@ -1391,12 +1362,19 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_decode_with_disktype", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Test that ec.decode accepts -diskType flag
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Defer unlock to ensure it's always released
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// Execute EC decode with disk type
|
||||
ecDecodeCmd := shell.Commands[findCommandIndex("ec.decode")]
|
||||
@@ -1442,7 +1420,6 @@ func startClusterWithDiskType(ctx context.Context, dataDir string, diskType stri
|
||||
"-mdir", masterDir,
|
||||
"-volumeSizeLimitMB", "10",
|
||||
"-ip", "127.0.0.1",
|
||||
"-peers", "none",
|
||||
)
|
||||
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
@@ -1588,7 +1565,12 @@ func TestECDiskTypeMixedCluster(t *testing.T) {
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
connectToMasterAndSync(ctx, t, commandEnv)
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
// Wait for master client to fully sync
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
t.Run("upload_to_ssd_and_hdd", func(t *testing.T) {
|
||||
// Upload to SSD
|
||||
@@ -1623,12 +1605,19 @@ func TestECDiskTypeMixedCluster(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_balance_targets_correct_disk_type", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock first
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Defer unlock to ensure it's always released
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// Run ec.balance for SSD collection with -diskType=ssd
|
||||
var ssdOutput bytes.Buffer
|
||||
@@ -1674,7 +1663,6 @@ func startMixedDiskTypeCluster(ctx context.Context, dataDir string) (*MultiDiskC
|
||||
"-mdir", masterDir,
|
||||
"-volumeSizeLimitMB", "10",
|
||||
"-ip", "127.0.0.1",
|
||||
"-peers", "none",
|
||||
)
|
||||
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
@@ -1776,7 +1764,11 @@ func TestEvacuationFallbackBehavior(t *testing.T) {
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
connectToMasterAndSync(ctx, t, commandEnv)
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
t.Run("fallback_when_same_disktype_full", func(t *testing.T) {
|
||||
// This test verifies that when evacuating SSD EC shards from a server,
|
||||
@@ -1801,12 +1793,18 @@ func TestEvacuationFallbackBehavior(t *testing.T) {
|
||||
|
||||
time.Sleep(3 * time.Second)
|
||||
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// EC encode the SSD volume
|
||||
var encodeOutput bytes.Buffer
|
||||
@@ -1870,7 +1868,11 @@ func TestCrossRackECPlacement(t *testing.T) {
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
connectToMasterAndSync(ctx, t, commandEnv)
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// Upload test data
|
||||
testData := []byte("Cross-rack EC placement test data - needs to be distributed across racks")
|
||||
@@ -1889,12 +1891,18 @@ func TestCrossRackECPlacement(t *testing.T) {
|
||||
time.Sleep(3 * time.Second)
|
||||
|
||||
t.Run("ec_encode_cross_rack", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// EC encode with rack-aware placement
|
||||
// Note: uploadTestDataToMaster uses collection "test" by default
|
||||
@@ -1939,12 +1947,18 @@ func TestCrossRackECPlacement(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_balance_respects_rack_placement", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
initialDistribution := countShardsPerRack(testDir, uint32(volumeId))
|
||||
t.Logf("Initial rack distribution: %v", initialDistribution)
|
||||
@@ -1992,7 +2006,6 @@ func startLimitedSsdCluster(ctx context.Context, dataDir string) (*MultiDiskClus
|
||||
"-mdir", masterDir,
|
||||
"-volumeSizeLimitMB", "10",
|
||||
"-ip", "127.0.0.1",
|
||||
"-peers", "none",
|
||||
)
|
||||
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
@@ -2083,7 +2096,6 @@ func startMultiRackCluster(ctx context.Context, dataDir string) (*MultiDiskClust
|
||||
"-mdir", masterDir,
|
||||
"-volumeSizeLimitMB", "10",
|
||||
"-ip", "127.0.0.1",
|
||||
"-peers", "none",
|
||||
)
|
||||
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
@@ -2187,249 +2199,3 @@ func countShardsPerRack(testDir string, volumeId uint32) map[string]int {
|
||||
|
||||
return rackDistribution
|
||||
}
|
||||
|
||||
// TestECEncodeReplicatedVolumeSync tests that ec.encode properly syncs missing entries
|
||||
// between replicas before encoding. This addresses issue #7797.
|
||||
func TestECEncodeReplicatedVolumeSync(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping replicated volume sync integration test in short mode")
|
||||
}
|
||||
|
||||
// Create temporary directory for test data
|
||||
testDir, err := os.MkdirTemp("", "seaweedfs_ec_replica_sync_test_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(testDir)
|
||||
|
||||
// Start SeaweedFS cluster
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startSeaweedFSCluster(ctx, testDir)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
// Wait for servers to be ready
|
||||
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
|
||||
for i := 0; i < 6; i++ {
|
||||
require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second))
|
||||
}
|
||||
|
||||
// Create command environment
|
||||
options := &shell.ShellOptions{
|
||||
Masters: stringPtr("127.0.0.1:9333"),
|
||||
GrpcDialOption: grpc.WithInsecure(),
|
||||
FilerGroup: stringPtr("default"),
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
// Connect to master with longer timeout
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel2()
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx2)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx2)
|
||||
|
||||
// Wait for volume servers to register with master
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// Test: Create replicated volume and verify sync behavior with consistent replicas
|
||||
t.Run("sync_replicated_volume_consistent", func(t *testing.T) {
|
||||
const testCollection = "replicated_consistent"
|
||||
const testReplication = "010" // 2 copies on different servers on the same rack
|
||||
|
||||
// Retry a few times as volume servers may still be registering
|
||||
var volumeId needle.VolumeId
|
||||
var uploadErr error
|
||||
for retry := 0; retry < 5; retry++ {
|
||||
volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
|
||||
if uploadErr == nil {
|
||||
break
|
||||
}
|
||||
t.Logf("Upload attempt %d failed: %v, retrying...", retry+1, uploadErr)
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
if uploadErr != nil {
|
||||
t.Skipf("Could not create replicated volume: %v", uploadErr)
|
||||
return
|
||||
}
|
||||
t.Logf("Created replicated volume %d with collection %s, replication %s", volumeId, testCollection, testReplication)
|
||||
|
||||
// Acquire lock
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout")
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Execute EC encoding with the same collection
|
||||
ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
|
||||
args := []string{
|
||||
"-volumeId", fmt.Sprintf("%d", volumeId),
|
||||
"-collection", testCollection,
|
||||
"-force",
|
||||
}
|
||||
|
||||
outputStr, encodeErr := captureCommandOutput(t, ecEncodeCmd, args, commandEnv)
|
||||
t.Logf("EC encode output:\n%s", outputStr)
|
||||
|
||||
// For consistent replicas, should see "all X replicas are consistent"
|
||||
assert.Contains(t, outputStr, "replicas are consistent", "Should detect replicas are consistent")
|
||||
|
||||
if encodeErr != nil {
|
||||
t.Logf("EC encoding result: %v", encodeErr)
|
||||
} else {
|
||||
t.Log("EC encoding completed successfully")
|
||||
}
|
||||
})
|
||||
|
||||
// Test: Create divergent replicas and verify sync/union code is exercised
|
||||
t.Run("sync_replicated_volume_divergent", func(t *testing.T) {
|
||||
const testCollection = "replicated_divergent"
|
||||
const testReplication = "010" // 2 copies on different servers on the same rack
|
||||
|
||||
// Create replicated volume with initial data
|
||||
var volumeId needle.VolumeId
|
||||
var uploadErr error
|
||||
for retry := 0; retry < 5; retry++ {
|
||||
volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
|
||||
if uploadErr == nil {
|
||||
break
|
||||
}
|
||||
t.Logf("Upload attempt %d failed: %v, retrying...", retry+1, uploadErr)
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
if uploadErr != nil {
|
||||
t.Skipf("Could not create replicated volume: %v", uploadErr)
|
||||
return
|
||||
}
|
||||
t.Logf("Created replicated volume %d with collection %s, replication %s", volumeId, testCollection, testReplication)
|
||||
|
||||
// Get volume locations to identify the two replicas
|
||||
locations, found := commandEnv.MasterClient.GetLocationsClone(uint32(volumeId))
|
||||
if !found || len(locations) < 2 {
|
||||
t.Skipf("Could not get 2 replica locations for volume %d (got %d)", volumeId, len(locations))
|
||||
return
|
||||
}
|
||||
t.Logf("Volume %d has replicas at: %v and %v", volumeId, locations[0].Url, locations[1].Url)
|
||||
|
||||
// Write an extra entry to ONLY one replica to create divergence
|
||||
// We'll use the WriteNeedleBlob gRPC call to inject data directly
|
||||
extraNeedleId := uint64(999999) // Use a high needle ID unlikely to conflict
|
||||
extraData := []byte("extra data written to only one replica to create divergence")
|
||||
|
||||
err := injectNeedleToOneReplica(grpc.WithInsecure(), volumeId, locations[0], extraNeedleId, extraData)
|
||||
if err != nil {
|
||||
t.Logf("Could not inject divergent needle (may not be supported): %v", err)
|
||||
// Fall back to testing consistent path
|
||||
} else {
|
||||
t.Logf("Injected extra needle %d to replica %s to create divergence", extraNeedleId, locations[0].Url)
|
||||
}
|
||||
|
||||
// Acquire lock
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout")
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Execute EC encoding - should detect divergence and build union
|
||||
ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
|
||||
args := []string{
|
||||
"-volumeId", fmt.Sprintf("%d", volumeId),
|
||||
"-collection", testCollection,
|
||||
"-force",
|
||||
}
|
||||
|
||||
outputStr, encodeErr := captureCommandOutput(t, ecEncodeCmd, args, commandEnv)
|
||||
t.Logf("EC encode output:\n%s", outputStr)
|
||||
|
||||
// Check if divergence was detected
|
||||
if strings.Contains(outputStr, "building union") {
|
||||
t.Log("SUCCESS: Divergent replicas detected - sync/union code was exercised")
|
||||
assert.Contains(t, outputStr, "selected", "Should show which replica was selected as best")
|
||||
// The "copied" message only appears if the best replica is missing entries from other replicas
|
||||
// In our case, we injected into what becomes the best replica (higher file count),
|
||||
// so copying may not be needed. The key is that "building union" was triggered.
|
||||
if strings.Contains(outputStr, "copied") {
|
||||
t.Log("Entries were copied between replicas")
|
||||
} else {
|
||||
t.Log("Best replica already had all entries (injected entry was on best replica)")
|
||||
}
|
||||
} else if strings.Contains(outputStr, "replicas are consistent") {
|
||||
t.Log("Replicas were consistent (injection may not have worked)")
|
||||
} else {
|
||||
t.Log("Single replica or sync not triggered")
|
||||
}
|
||||
|
||||
if encodeErr != nil {
|
||||
t.Logf("EC encoding result: %v", encodeErr)
|
||||
} else {
|
||||
t.Log("EC encoding completed successfully")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// injectNeedleToOneReplica writes a needle directly to one replica to create divergence
|
||||
func injectNeedleToOneReplica(grpcDialOption grpc.DialOption, vid needle.VolumeId, location wdclient.Location, needleId uint64, data []byte) error {
|
||||
return operation.WithVolumeServerClient(false, location.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
_, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{
|
||||
VolumeId: uint32(vid),
|
||||
NeedleId: needleId,
|
||||
Size: int32(len(data)),
|
||||
NeedleBlob: data,
|
||||
})
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
// uploadTestDataWithReplication uploads test data and returns the volume ID
|
||||
// using the specified collection and replication level. All files are uploaded to the same volume.
|
||||
func uploadTestDataWithReplication(t *testing.T, masterAddr string, collection string, replication string) (needle.VolumeId, error) {
|
||||
const numFiles = 20 // Reduced count since we're uploading to same volume
|
||||
|
||||
// Assign multiple file IDs from the same volume
|
||||
assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
|
||||
return pb.ServerAddress(masterAddr)
|
||||
}, grpc.WithInsecure(), &operation.VolumeAssignRequest{
|
||||
Count: uint64(numFiles),
|
||||
Collection: collection,
|
||||
Replication: replication,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to assign volume: %v", err)
|
||||
}
|
||||
|
||||
// Parse volume ID from file ID
|
||||
fid, err := needle.ParseFileIdFromString(assignResult.Fid)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to parse file ID: %v", err)
|
||||
}
|
||||
volumeId := fid.VolumeId
|
||||
|
||||
// Create uploader for upload
|
||||
uploader, err := operation.NewUploader()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to create uploader: %v", err)
|
||||
}
|
||||
|
||||
// Upload test data to the same volume using the assigned file IDs
|
||||
// The first FID is assignResult.Fid, subsequent ones increment the needle key
|
||||
baseNeedleKey := uint64(fid.Key)
|
||||
for i := 0; i < numFiles; i++ {
|
||||
data := []byte(fmt.Sprintf("test data for replicated volume sync test - entry %d - padding to make it larger %s",
|
||||
i, strings.Repeat("x", 1000)))
|
||||
|
||||
// Construct FID for this file (same volume, incrementing needle key)
|
||||
currentFid := fmt.Sprintf("%d,%x%08x", volumeId, baseNeedleKey+uint64(i), fid.Cookie)
|
||||
uploadUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, currentFid)
|
||||
|
||||
_, _, uploadErr := uploader.Upload(context.Background(), bytes.NewReader(data), &operation.UploadOption{
|
||||
UploadUrl: uploadUrl,
|
||||
Filename: fmt.Sprintf("file%d.txt", i),
|
||||
})
|
||||
if uploadErr != nil {
|
||||
t.Logf("Upload %d failed: %v", i, uploadErr)
|
||||
}
|
||||
}
|
||||
|
||||
return volumeId, nil
|
||||
}
|
||||
|
||||
@@ -5,14 +5,13 @@ ARG FOUNDATIONDB_VERSION=7.4.5
|
||||
ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}
|
||||
|
||||
# Install build dependencies and download prebuilt FoundationDB clients
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN set -euo pipefail && \
|
||||
apt-get update && apt-get install -y \
|
||||
RUN apt-get update && apt-get install -y \
|
||||
build-essential \
|
||||
git \
|
||||
wget \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/* && \
|
||||
set -euo pipefail && \
|
||||
case "${FOUNDATIONDB_VERSION}" in \
|
||||
"7.4.5") EXPECTED_SHA256="f2176b86b7e1b561c3632b4e6e7efb82e3b8f57c2ff0d0ac4671e742867508aa" ;; \
|
||||
*) echo "ERROR: No known ARM64 client checksum for FoundationDB ${FOUNDATIONDB_VERSION}. Please update this Dockerfile." >&2; exit 1 ;; \
|
||||
|
||||
@@ -15,7 +15,6 @@ RUN apt-get update && apt-get install -y \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install FoundationDB server + client debs with checksum verification
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN set -euo pipefail && \
|
||||
apt-get update && \
|
||||
case "${FOUNDATIONDB_VERSION}" in \
|
||||
|
||||
@@ -50,24 +50,6 @@ test-benchmark: ## Run performance benchmarks
|
||||
@echo "$(YELLOW)Running FoundationDB performance benchmarks...$(NC)"
|
||||
@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb -bench=. ./test/foundationdb/...
|
||||
|
||||
test-benchmark-filer: ## Run filer store benchmarks (batch vs no-batch comparison)
|
||||
@echo "$(YELLOW)Running FoundationDB filer store benchmarks...$(NC)"
|
||||
@echo "$(BLUE)Comparing batched vs non-batched write performance$(NC)"
|
||||
@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb \
|
||||
-bench='BenchmarkFoundationDBStore_.*' \
|
||||
-benchmem \
|
||||
-benchtime=5s \
|
||||
./weed/filer/foundationdb/...
|
||||
|
||||
test-benchmark-concurrent: ## Run concurrent operation benchmarks
|
||||
@echo "$(YELLOW)Running concurrent operation benchmarks...$(NC)"
|
||||
@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb \
|
||||
-bench='BenchmarkFoundationDBStore_Concurrent.*' \
|
||||
-benchmem \
|
||||
-benchtime=10s \
|
||||
-cpu=1,2,4,8 \
|
||||
./weed/filer/foundationdb/...
|
||||
|
||||
# ARM64 specific targets (Apple Silicon / M1/M2/M3 Macs)
|
||||
setup-arm64: ## Set up ARM64-native FoundationDB cluster (builds from source)
|
||||
@echo "$(YELLOW)Setting up ARM64-native FoundationDB cluster...$(NC)"
|
||||
|
||||
@@ -353,31 +353,12 @@ The tests are designed to be reliable in CI environments with:
|
||||
Run performance benchmarks:
|
||||
|
||||
```bash
|
||||
make test-benchmark # Run all benchmarks
|
||||
make test-benchmark-filer # Filer store benchmarks with batch comparison
|
||||
make test-benchmark-concurrent # Concurrent operation benchmarks (varies CPU count)
|
||||
make test-benchmark
|
||||
|
||||
# Sample expected results:
|
||||
# BenchmarkFoundationDBStore_InsertEntry-8 1000 1.2ms per op
|
||||
# BenchmarkFoundationDBStore_FindEntry-8 5000 0.5ms per op
|
||||
# BenchmarkFoundationDBStore_KvOperations-8 2000 0.8ms per op
|
||||
# BenchmarkFoundationDBStore_InsertEntry_NoBatch-8 1000 1.0ms per op (optimal for S3)
|
||||
# BenchmarkFoundationDBStore_InsertEntry_WithBatch-8 1200 0.9ms per op (bulk ingestion)
|
||||
# BenchmarkFoundationDBStore_ConcurrentInsert_*-8 5000 0.3ms per op (parallel writes)
|
||||
```
|
||||
|
||||
### Batch vs Non-Batch Performance
|
||||
|
||||
The FoundationDB filer store supports two write modes:
|
||||
|
||||
| Mode | Config | Best For | Latency | Throughput |
|
||||
|------|--------|----------|---------|------------|
|
||||
| **Direct Commit** (default) | `batch_enabled = false` | S3 API, low-latency workloads | ~1-5ms per op | Good |
|
||||
| **Batched** | `batch_enabled = true` | Bulk ingestion, high throughput | Variable | Higher |
|
||||
|
||||
Run the comparison benchmark:
|
||||
```bash
|
||||
make test-benchmark-filer
|
||||
# BenchmarkFoundationDBStore_InsertEntry-8 1000 1.2ms per op
|
||||
# BenchmarkFoundationDBStore_FindEntry-8 5000 0.5ms per op
|
||||
# BenchmarkFoundationDBStore_KvOperations-8 2000 0.8ms per op
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
@@ -28,8 +28,8 @@ services:
|
||||
echo 'testing:testing@fdb1:4500,fdb2:4502,fdb3:4504' > /var/fdb/config/fdb.cluster
|
||||
fi
|
||||
# Start FDB processes
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
|
||||
wait
|
||||
"
|
||||
|
||||
@@ -59,8 +59,8 @@ services:
|
||||
# Wait for cluster file from fdb1
|
||||
while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
|
||||
# Start FDB processes
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
|
||||
wait
|
||||
"
|
||||
|
||||
@@ -90,8 +90,8 @@ services:
|
||||
# Wait for cluster file from fdb1
|
||||
while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
|
||||
# Start FDB processes
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
|
||||
wait
|
||||
"
|
||||
|
||||
|
||||
@@ -1,8 +1,19 @@
|
||||
# FoundationDB Filer Configuration for Testing
|
||||
# FoundationDB Filer Configuration
|
||||
|
||||
[foundationdb]
|
||||
enabled = true
|
||||
cluster_file = "/var/fdb/config/fdb.cluster"
|
||||
# api_version = 740
|
||||
# timeout = "5s"
|
||||
# directory_prefix = "seaweedfs"
|
||||
api_version = 740
|
||||
timeout = "5s"
|
||||
max_retry_delay = "1s"
|
||||
directory_prefix = "seaweedfs"
|
||||
|
||||
# For testing different configurations
|
||||
[foundationdb.test]
|
||||
enabled = false
|
||||
cluster_file = "/var/fdb/config/fdb.cluster"
|
||||
api_version = 740
|
||||
timeout = "10s"
|
||||
max_retry_delay = "2s"
|
||||
directory_prefix = "seaweedfs_test"
|
||||
location = "/test"
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
<junit.version>4.13.2</junit.version>
|
||||
<seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
|
||||
<jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
|
||||
<netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
|
||||
<netty.version>4.1.125.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
|
||||
<parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
|
||||
<parquet.format.version>2.12.0</parquet.format.version>
|
||||
<surefire.jvm.args>
|
||||
|
||||
@@ -172,8 +172,7 @@ services:
|
||||
- -master=seaweedfs-master:9333
|
||||
- -ip=seaweedfs-mq-broker
|
||||
- -port=17777
|
||||
- -debug
|
||||
- -debug.port=18777
|
||||
- -port.pprof=18777
|
||||
depends_on:
|
||||
seaweedfs-filer:
|
||||
condition: service_healthy
|
||||
|
||||
+22
-22
@@ -43,25 +43,25 @@ require (
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go v1.55.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
|
||||
github.com/aws/smithy-go v1.23.2 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
@@ -229,15 +229,15 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.46.0 // indirect
|
||||
golang.org/x/crypto v0.45.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect
|
||||
golang.org/x/image v0.34.0 // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/image v0.33.0 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/oauth2 v0.32.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/term v0.38.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/term v0.37.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
google.golang.org/api v0.247.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
|
||||
|
||||
+54
-50
@@ -102,44 +102,44 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.5 h1:xMo63RlqP3ZZydpJDMBsH9uJ10hgHYfQFIk1cHDXrR4=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.5/go.mod h1:hhbH6oRcou+LpXfA/0vPElh/e0M3aFeOblE1sssAAEk=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 h1:eYnlt6QxnFINKzwxP5/Ucs1vkG7VT3Iezmvfgc2waUw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.7/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
|
||||
github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
|
||||
github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo=
|
||||
@@ -536,10 +536,10 @@ github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4
|
||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
|
||||
github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
|
||||
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
|
||||
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
|
||||
github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg=
|
||||
github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
|
||||
github.com/rclone/rclone v1.71.2 h1:3Jk5xNPFrZhVABRuN/OPvApuZQddpE2tkhYMuEn1Ud4=
|
||||
github.com/rclone/rclone v1.71.2/go.mod h1:dCK9FzPDlpkbQJ9M7MmWsmv3X5nibfWe+ogJXu6gSgM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
|
||||
@@ -708,6 +708,8 @@ go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJr
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
|
||||
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
@@ -730,8 +732,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
|
||||
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
|
||||
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
|
||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -746,8 +748,8 @@ golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 h1:3yiSh9fhy5/RhCSntf4Sy0Tnx
|
||||
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.34.0 h1:33gCkyw9hmwbZJeZkct8XyR11yH889EQt/QH4VmXMn8=
|
||||
golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
|
||||
golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
|
||||
golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -772,6 +774,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
|
||||
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -811,8 +815,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -890,8 +894,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -904,8 +908,8 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
|
||||
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
|
||||
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
|
||||
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
|
||||
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -921,8 +925,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@@ -975,8 +979,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
|
||||
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
|
||||
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
|
||||
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
|
||||
@@ -181,8 +181,7 @@ services:
|
||||
- -ip=seaweedfs-mq-broker
|
||||
- -port=17777
|
||||
- -logFlushInterval=0
|
||||
- -debug
|
||||
- -debug.port=18777
|
||||
- -port.pprof=18777
|
||||
depends_on:
|
||||
seaweedfs-filer:
|
||||
condition: service_healthy
|
||||
|
||||
@@ -25,4 +25,3 @@ test-resume: build-weed
|
||||
clean:
|
||||
rm -f ../../weed/weed
|
||||
|
||||
|
||||
|
||||
@@ -47,4 +47,3 @@ go test -v -timeout 5m ./test/metadata_subscribe/...
|
||||
- Tests create temporary directories that are cleaned up after completion
|
||||
- Tests use ports 9333 (master), 8080 (volume), 8888 (filer)
|
||||
|
||||
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
# S3 SDK Compatibility Testing
|
||||
|
||||
This document describes the SDK versions tested against SeaweedFS S3 API and known compatibility considerations.
|
||||
|
||||
## Validated SDK Versions
|
||||
|
||||
### Go SDKs
|
||||
|
||||
| SDK | Version | Test Location | Status |
|
||||
|-----|---------|---------------|--------|
|
||||
| AWS SDK v2 for Go | 1.20+ | `test/s3/etag/`, `test/s3/copying/` | ✅ Tested |
|
||||
| AWS SDK v1 for Go | 1.x | `test/s3/basic/` | ✅ Tested |
|
||||
|
||||
### Java SDKs
|
||||
|
||||
| SDK | Version | Test Location | Status |
|
||||
|-----|---------|---------------|--------|
|
||||
| AWS SDK v2 for Java | 2.20.127+ | `other/java/s3copier/` | ✅ Tested |
|
||||
| AWS SDK v1 for Java | 1.12.600+ | `other/java/s3copier/` | ✅ Tested |
|
||||
|
||||
### Python SDKs
|
||||
|
||||
| SDK | Version | Test Location | Status |
|
||||
|-----|---------|---------------|--------|
|
||||
| boto3 | 1.x | `test/s3/parquet/` | ✅ Tested |
|
||||
| PyArrow S3 | 14+ | `test/s3/parquet/` | ✅ Tested |
|
||||
|
||||
## SDK-Specific Considerations
|
||||
|
||||
### AWS SDK v2 for Java - ETag Validation
|
||||
|
||||
**Issue**: [GitHub #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
|
||||
|
||||
AWS SDK v2 for Java includes strict ETag validation in `ChecksumsEnabledValidator.validatePutObjectChecksum`. It decodes the ETag as a hexadecimal MD5 hash using `Base16Codec.decode()`.
|
||||
|
||||
**Impact**: If the ETag contains non-hexadecimal characters (like `-` in composite format), the SDK fails with:
|
||||
```text
|
||||
java.lang.IllegalArgumentException: Invalid base 16 character: '-'
|
||||
```
|
||||
|
||||
**Resolution**: SeaweedFS now correctly returns:
|
||||
- **PutObject**: Pure MD5 hex ETag (32 characters) regardless of internal chunking
|
||||
- **CompleteMultipartUpload**: Composite ETag (`<md5>-<partcount>`)
|
||||
|
||||
**Test Coverage**: `test/s3/etag/` and `other/java/s3copier/ETagValidationTest.java`
|
||||
|
||||
### AWS SDK v1 vs v2 Differences
|
||||
|
||||
| Feature | SDK v1 | SDK v2 |
|
||||
|---------|--------|--------|
|
||||
| ETag hex validation | No | Yes (strict) |
|
||||
| Checksum validation | Basic | Enhanced |
|
||||
| Async support | Limited | Full |
|
||||
| Default retry behavior | Lenient | Stricter |
|
||||
|
||||
### Large File Handling
|
||||
|
||||
SeaweedFS auto-chunks files larger than **8MB** for efficient storage. This is transparent to clients, but previously affected ETag format. The current implementation ensures:
|
||||
|
||||
1. Regular `PutObject` (any size): Returns pure MD5 ETag
|
||||
2. Multipart upload: Returns composite ETag per AWS S3 specification
|
||||
|
||||
## Test Categories by File Size
|
||||
|
||||
| Category | Size | Chunks | ETag Format |
|
||||
|----------|------|--------|-------------|
|
||||
| Small | < 256KB | 1 (inline) | Pure MD5 |
|
||||
| Medium | 256KB - 8MB | 1 | Pure MD5 |
|
||||
| Large | 8MB - 24MB | 2-3 | Pure MD5 |
|
||||
| Extra Large | > 24MB | 4+ | Pure MD5 |
|
||||
| Multipart | N/A | Per part | Composite |
|
||||
|
||||
## Running SDK Compatibility Tests
|
||||
|
||||
### Go Tests
|
||||
|
||||
```bash
|
||||
# Run all ETag tests
|
||||
cd test/s3/etag && make test
|
||||
|
||||
# Run large file tests only
|
||||
cd test/s3/etag && make test-large
|
||||
```
|
||||
|
||||
### Java Tests
|
||||
|
||||
```bash
|
||||
# Run all Java SDK tests
|
||||
cd other/java/s3copier && mvn test
|
||||
|
||||
# Run only ETag validation tests
|
||||
cd other/java/s3copier && mvn test -Dtest=ETagValidationTest
|
||||
```
|
||||
|
||||
### Python Tests
|
||||
|
||||
```bash
|
||||
# Run PyArrow S3 tests
|
||||
cd test/s3/parquet && make test
|
||||
```
|
||||
|
||||
## Adding New SDK Tests
|
||||
|
||||
When adding tests for new SDKs, ensure:
|
||||
|
||||
1. **Large file tests (>8MB)**: Critical for verifying ETag format with auto-chunking
|
||||
2. **Multipart upload tests**: Verify composite ETag format
|
||||
3. **Checksum validation**: Test SDK-specific checksum validation if applicable
|
||||
4. **Document SDK version**: Add to this compatibility matrix
|
||||
|
||||
## Known Issues and Workarounds
|
||||
|
||||
### Issue: Older SDK Versions
|
||||
|
||||
Some very old SDK versions (e.g., AWS SDK v1 for Java < 1.11.x) may have different behavior. Testing with the versions listed above is recommended.
|
||||
|
||||
### Issue: Custom Checksum Algorithms
|
||||
|
||||
AWS SDK v2 supports SHA-256 and CRC32 checksums in addition to MD5. SeaweedFS currently returns MD5-based ETags. For checksums other than MD5, use the `x-amz-checksum-*` headers.
|
||||
|
||||
## References
|
||||
|
||||
- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
|
||||
- [AWS SDK v2 Migration Guide](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/migration.html)
|
||||
- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# ETag Format Integration Tests
|
||||
#
|
||||
# These tests verify S3 ETag format compatibility, particularly for large files
|
||||
# that trigger SeaweedFS auto-chunking. This addresses GitHub Issue #7768.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - SeaweedFS running with S3 API enabled on port 8333
|
||||
# - Go 1.21+
|
||||
#
|
||||
# Usage:
|
||||
# make test - Run all tests
|
||||
# make test-large - Run only large file tests
|
||||
# make test-verbose - Run with verbose output
|
||||
# make clean - Clean test artifacts
|
||||
|
||||
.PHONY: all test test-large test-verbose test-quick clean help
|
||||
|
||||
# Default S3 endpoint
|
||||
S3_ENDPOINT ?= http://127.0.0.1:8333
|
||||
|
||||
all: test
|
||||
|
||||
test:
|
||||
@echo "Running ETag format tests against $(S3_ENDPOINT)..."
|
||||
S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m ./...
|
||||
|
||||
test-large:
|
||||
@echo "Running large file ETag tests..."
|
||||
S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -run "LargeFile|ExtraLarge" ./...
|
||||
|
||||
test-verbose:
|
||||
S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 5m -count=1 ./...
|
||||
|
||||
test-quick:
|
||||
@echo "Running quick ETag tests (small files only)..."
|
||||
S3_ENDPOINT=$(S3_ENDPOINT) go test -v -timeout 1m -run "SmallFile|Consistency" ./...
|
||||
|
||||
clean:
|
||||
go clean -testcache
|
||||
|
||||
help:
|
||||
@echo "ETag Format Integration Tests"
|
||||
@echo "Targets:"
|
||||
@echo " test Run all ETag format tests"
|
||||
@echo " test-large Run only large file tests (>8MB)"
|
||||
@echo " test-quick Run quick tests (small files only)"
|
||||
@echo " test-verbose Run with verbose output"
|
||||
@echo " clean Clean test cache"
|
||||
@echo "Environment Variables:"
|
||||
@echo " S3_ENDPOINT S3 endpoint URL (default: http://127.0.0.1:8333)"
|
||||
@@ -1,93 +0,0 @@
|
||||
# S3 ETag Format Integration Tests
|
||||
|
||||
This test suite verifies that SeaweedFS returns correct ETag formats for S3 operations, ensuring compatibility with AWS S3 SDKs.
|
||||
|
||||
## Background
|
||||
|
||||
**GitHub Issue #7768**: AWS S3 SDK for Java v2 was failing with `Invalid base 16 character: '-'` when performing `PutObject` on large files.
|
||||
|
||||
### Root Cause
|
||||
|
||||
SeaweedFS internally auto-chunks large files (>8MB) for efficient storage. Previously, when a regular `PutObject` request resulted in multiple internal chunks, SeaweedFS returned a composite ETag format (`<md5>-<count>`) instead of a pure MD5 hash.
|
||||
|
||||
### AWS S3 Specification
|
||||
|
||||
| Operation | ETag Format | Example |
|
||||
|-----------|-------------|---------|
|
||||
| PutObject (any size) | Pure MD5 hex (32 chars) | `d41d8cd98f00b204e9800998ecf8427e` |
|
||||
| CompleteMultipartUpload | Composite (`<md5>-<partcount>`) | `d41d8cd98f00b204e9800998ecf8427e-3` |
|
||||
|
||||
AWS S3 SDK v2 for Java validates `PutObject` ETags as hexadecimal, which fails when the ETag contains a hyphen.
|
||||
|
||||
## Test Coverage
|
||||
|
||||
| Test | File Size | Purpose |
|
||||
|------|-----------|---------|
|
||||
| `TestPutObjectETagFormat_SmallFile` | 1KB | Verify single-chunk uploads return pure MD5 |
|
||||
| `TestPutObjectETagFormat_LargeFile` | 10MB | **Critical**: Verify auto-chunked uploads return pure MD5 |
|
||||
| `TestPutObjectETagFormat_ExtraLargeFile` | 25MB | Verify multi-chunk auto-chunked uploads return pure MD5 |
|
||||
| `TestMultipartUploadETagFormat` | 15MB | Verify multipart uploads correctly return composite ETag |
|
||||
| `TestPutObjectETagConsistency` | Various | Verify ETag consistency across PUT/HEAD/GET |
|
||||
| `TestETagHexValidation` | 10MB | Simulate AWS SDK v2 hex validation |
|
||||
| `TestMultipleLargeFileUploads` | 10MB x5 | Stress test multiple large uploads |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. SeaweedFS running with S3 API enabled:
|
||||
```bash
|
||||
weed server -s3
|
||||
```
|
||||
|
||||
2. Go 1.21 or later
|
||||
|
||||
3. AWS SDK v2 for Go (installed via go modules)
|
||||
|
||||
## Running Tests
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
make test
|
||||
|
||||
# Run only large file tests (the critical ones for issue #7768)
|
||||
make test-large
|
||||
|
||||
# Run quick tests (small files only)
|
||||
make test-quick
|
||||
|
||||
# Run with verbose output
|
||||
make test-verbose
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
By default, tests connect to `http://127.0.0.1:8333`. To use a different endpoint:
|
||||
|
||||
```bash
|
||||
S3_ENDPOINT=http://localhost:8333 make test
|
||||
```
|
||||
|
||||
Or modify `defaultConfig` in `s3_etag_test.go`.
|
||||
|
||||
## SDK Compatibility
|
||||
|
||||
These tests use **AWS SDK v2 for Go**, which has the same ETag validation behavior as AWS SDK v2 for Java. The tests include:
|
||||
|
||||
- ETag format validation (pure MD5 vs composite)
|
||||
- Hex decoding validation (simulates `Base16Codec.decode`)
|
||||
- Content integrity verification
|
||||
|
||||
## Validated SDK Versions
|
||||
|
||||
| SDK | Version | Status |
|
||||
|-----|---------|--------|
|
||||
| AWS SDK v2 for Go | 1.20+ | ✅ Tested |
|
||||
| AWS SDK v2 for Java | 2.20+ | ✅ Compatible (issue #7768 fixed) |
|
||||
| AWS SDK v1 for Go | 1.x | ✅ Compatible (less strict validation) |
|
||||
| AWS SDK v1 for Java | 1.x | ✅ Compatible (less strict validation) |
|
||||
|
||||
## Related
|
||||
|
||||
- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
|
||||
- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
|
||||
|
||||
|
||||
@@ -1,543 +0,0 @@
|
||||
// Package etag_test provides integration tests for S3 ETag format validation.
|
||||
//
|
||||
// These tests verify that SeaweedFS returns correct ETag formats for different
|
||||
// upload scenarios, ensuring compatibility with AWS S3 SDKs that validate ETags.
|
||||
//
|
||||
// Background (GitHub Issue #7768):
|
||||
// AWS S3 SDK for Java v2 validates ETags as hexadecimal MD5 hashes for PutObject
|
||||
// responses. SeaweedFS was incorrectly returning composite ETags ("<md5>-<count>")
|
||||
// for regular PutObject when files were internally auto-chunked (>8MB), causing
|
||||
// the SDK to fail with "Invalid base 16 character: '-'".
|
||||
//
|
||||
// Per AWS S3 specification:
|
||||
// - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
|
||||
// - Multipart Upload (CompleteMultipartUpload): ETag is "<md5>-<partcount>"
|
||||
//
|
||||
// These tests ensure this behavior is maintained.
|
||||
package etag_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
mathrand "math/rand"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// S3TestConfig holds configuration for S3 tests
|
||||
type S3TestConfig struct {
|
||||
Endpoint string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
Region string
|
||||
BucketPrefix string
|
||||
}
|
||||
|
||||
// Default test configuration
|
||||
var defaultConfig = &S3TestConfig{
|
||||
Endpoint: "http://127.0.0.1:8333",
|
||||
AccessKey: "some_access_key1",
|
||||
SecretKey: "some_secret_key1",
|
||||
Region: "us-east-1",
|
||||
BucketPrefix: "test-etag-",
|
||||
}
|
||||
|
||||
// Constants for auto-chunking thresholds (must match s3api_object_handlers_put.go)
|
||||
const (
|
||||
// SeaweedFS auto-chunks files larger than 8MB
|
||||
autoChunkSize = 8 * 1024 * 1024
|
||||
|
||||
// Test sizes
|
||||
smallFileSize = 1 * 1024 // 1KB - single chunk
|
||||
mediumFileSize = 256 * 1024 // 256KB - single chunk (at threshold)
|
||||
largeFileSize = 10 * 1024 * 1024 // 10MB - triggers auto-chunking (2 chunks)
|
||||
xlFileSize = 25 * 1024 * 1024 // 25MB - triggers auto-chunking (4 chunks)
|
||||
multipartSize = 5 * 1024 * 1024 // 5MB per part for multipart uploads
|
||||
)
|
||||
|
||||
// ETag format patterns
|
||||
var (
|
||||
// Pure MD5 ETag: 32 hex characters (with or without quotes)
|
||||
pureMD5Pattern = regexp.MustCompile(`^"?[a-f0-9]{32}"?$`)
|
||||
|
||||
// Composite ETag for multipart: 32 hex chars, hyphen, part count (with or without quotes)
|
||||
compositePattern = regexp.MustCompile(`^"?[a-f0-9]{32}-\d+"?$`)
|
||||
)
|
||||
|
||||
func init() {
|
||||
mathrand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// getS3Client creates an AWS S3 v2 client for testing
|
||||
func getS3Client(t *testing.T) *s3.Client {
|
||||
cfg, err := config.LoadDefaultConfig(context.TODO(),
|
||||
config.WithRegion(defaultConfig.Region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
defaultConfig.AccessKey,
|
||||
defaultConfig.SecretKey,
|
||||
"",
|
||||
)),
|
||||
config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
|
||||
func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
URL: defaultConfig.Endpoint,
|
||||
SigningRegion: defaultConfig.Region,
|
||||
HostnameImmutable: true,
|
||||
}, nil
|
||||
})),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.UsePathStyle = true
|
||||
})
|
||||
}
|
||||
|
||||
// getNewBucketName generates a unique bucket name
|
||||
func getNewBucketName() string {
|
||||
timestamp := time.Now().UnixNano()
|
||||
randomSuffix := mathrand.Intn(100000)
|
||||
return fmt.Sprintf("%s%d-%d", defaultConfig.BucketPrefix, timestamp, randomSuffix)
|
||||
}
|
||||
|
||||
// generateRandomData generates random test data of specified size
|
||||
func generateRandomData(size int) []byte {
|
||||
data := make([]byte, size)
|
||||
if _, err := rand.Read(data); err != nil {
|
||||
panic(fmt.Sprintf("failed to generate random test data: %v", err))
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// calculateMD5 calculates the MD5 hash of data and returns hex string
|
||||
func calculateMD5(data []byte) string {
|
||||
hash := md5.Sum(data)
|
||||
return hex.EncodeToString(hash[:])
|
||||
}
|
||||
|
||||
// cleanETag removes quotes from ETag
|
||||
func cleanETag(etag string) string {
|
||||
return strings.Trim(etag, `"`)
|
||||
}
|
||||
|
||||
// isPureMD5ETag checks if ETag is a pure MD5 hex string (no composite format)
|
||||
func isPureMD5ETag(etag string) bool {
|
||||
return pureMD5Pattern.MatchString(etag)
|
||||
}
|
||||
|
||||
// isCompositeETag checks if ETag is in composite format (md5-count)
|
||||
func isCompositeETag(etag string) bool {
|
||||
return compositePattern.MatchString(etag)
|
||||
}
|
||||
|
||||
// createTestBucket creates a new bucket for testing
|
||||
func createTestBucket(ctx context.Context, client *s3.Client, bucketName string) error {
|
||||
_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// cleanupTestBucket deletes all objects and the bucket
|
||||
func cleanupTestBucket(ctx context.Context, client *s3.Client, bucketName string) {
|
||||
// Delete all objects
|
||||
paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
|
||||
for paginator.HasMorePages() {
|
||||
page, err := paginator.NextPage(ctx)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
for _, obj := range page.Contents {
|
||||
client.DeleteObject(ctx, &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: obj.Key,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Abort any in-progress multipart uploads
|
||||
mpPaginator := s3.NewListMultipartUploadsPaginator(client, &s3.ListMultipartUploadsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
for mpPaginator.HasMorePages() {
|
||||
page, err := mpPaginator.NextPage(ctx)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
for _, upload := range page.Uploads {
|
||||
client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: upload.Key,
|
||||
UploadId: upload.UploadId,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Delete bucket
|
||||
client.DeleteBucket(ctx, &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
}
|
||||
|
||||
// TestPutObjectETagFormat_SmallFile verifies ETag format for small files (single chunk)
|
||||
func TestPutObjectETagFormat_SmallFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := getS3Client(t)
|
||||
|
||||
bucketName := getNewBucketName()
|
||||
err := createTestBucket(ctx, client, bucketName)
|
||||
require.NoError(t, err, "Failed to create test bucket")
|
||||
defer cleanupTestBucket(ctx, client, bucketName)
|
||||
|
||||
testData := generateRandomData(smallFileSize)
|
||||
expectedMD5 := calculateMD5(testData)
|
||||
objectKey := "small-file.bin"
|
||||
|
||||
// Upload small file
|
||||
putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(testData),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload small file")
|
||||
|
||||
// Verify ETag format
|
||||
etag := aws.ToString(putResp.ETag)
|
||||
t.Logf("Small file (%d bytes) ETag: %s", smallFileSize, etag)
|
||||
|
||||
assert.True(t, isPureMD5ETag(etag),
|
||||
"Small file ETag should be pure MD5, got: %s", etag)
|
||||
assert.False(t, isCompositeETag(etag),
|
||||
"Small file ETag should NOT be composite format, got: %s", etag)
|
||||
assert.Equal(t, expectedMD5, cleanETag(etag),
|
||||
"ETag should match calculated MD5")
|
||||
}
|
||||
|
||||
// TestPutObjectETagFormat_LargeFile verifies ETag format for large files that trigger auto-chunking
|
||||
// This is the critical test for GitHub Issue #7768
|
||||
func TestPutObjectETagFormat_LargeFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := getS3Client(t)
|
||||
|
||||
bucketName := getNewBucketName()
|
||||
err := createTestBucket(ctx, client, bucketName)
|
||||
require.NoError(t, err, "Failed to create test bucket")
|
||||
defer cleanupTestBucket(ctx, client, bucketName)
|
||||
|
||||
testData := generateRandomData(largeFileSize)
|
||||
expectedMD5 := calculateMD5(testData)
|
||||
objectKey := "large-file.bin"
|
||||
|
||||
t.Logf("Uploading large file (%d bytes, > %d byte auto-chunk threshold)...",
|
||||
largeFileSize, autoChunkSize)
|
||||
|
||||
// Upload large file (triggers auto-chunking internally)
|
||||
putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(testData),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload large file")
|
||||
|
||||
// Verify ETag format - MUST be pure MD5, NOT composite
|
||||
etag := aws.ToString(putResp.ETag)
|
||||
t.Logf("Large file (%d bytes, ~%d internal chunks) ETag: %s",
|
||||
largeFileSize, (largeFileSize/autoChunkSize)+1, etag)
|
||||
|
||||
assert.True(t, isPureMD5ETag(etag),
|
||||
"Large file PutObject ETag MUST be pure MD5 (not composite), got: %s", etag)
|
||||
assert.False(t, isCompositeETag(etag),
|
||||
"Large file PutObject ETag should NOT contain '-' (composite format), got: %s", etag)
|
||||
assert.False(t, strings.Contains(cleanETag(etag), "-"),
|
||||
"ETag should not contain hyphen for regular PutObject, got: %s", etag)
|
||||
assert.Equal(t, expectedMD5, cleanETag(etag),
|
||||
"ETag should match calculated MD5 of entire content")
|
||||
|
||||
// Verify we can read back the object correctly
|
||||
getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to get large file")
|
||||
defer getResp.Body.Close()
|
||||
|
||||
downloadedData, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err, "Failed to read large file content")
|
||||
assert.Equal(t, testData, downloadedData, "Downloaded content should match uploaded content")
|
||||
}
|
||||
|
||||
// TestPutObjectETagFormat_ExtraLargeFile tests even larger files with multiple internal chunks
|
||||
func TestPutObjectETagFormat_ExtraLargeFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := getS3Client(t)
|
||||
|
||||
bucketName := getNewBucketName()
|
||||
err := createTestBucket(ctx, client, bucketName)
|
||||
require.NoError(t, err, "Failed to create test bucket")
|
||||
defer cleanupTestBucket(ctx, client, bucketName)
|
||||
|
||||
testData := generateRandomData(xlFileSize)
|
||||
expectedMD5 := calculateMD5(testData)
|
||||
objectKey := "xl-file.bin"
|
||||
|
||||
expectedChunks := (xlFileSize / autoChunkSize) + 1
|
||||
t.Logf("Uploading XL file (%d bytes, expected ~%d internal chunks)...",
|
||||
xlFileSize, expectedChunks)
|
||||
|
||||
// Upload extra large file
|
||||
putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(testData),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload XL file")
|
||||
|
||||
// Verify ETag format
|
||||
etag := aws.ToString(putResp.ETag)
|
||||
t.Logf("XL file (%d bytes) ETag: %s", xlFileSize, etag)
|
||||
|
||||
assert.True(t, isPureMD5ETag(etag),
|
||||
"XL file PutObject ETag MUST be pure MD5, got: %s", etag)
|
||||
assert.False(t, isCompositeETag(etag),
|
||||
"XL file PutObject ETag should NOT be composite, got: %s", etag)
|
||||
assert.Equal(t, expectedMD5, cleanETag(etag),
|
||||
"ETag should match calculated MD5")
|
||||
}
|
||||
|
||||
// TestMultipartUploadETagFormat verifies that ONLY multipart uploads get composite ETags
|
||||
func TestMultipartUploadETagFormat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := getS3Client(t)
|
||||
|
||||
bucketName := getNewBucketName()
|
||||
err := createTestBucket(ctx, client, bucketName)
|
||||
require.NoError(t, err, "Failed to create test bucket")
|
||||
defer cleanupTestBucket(ctx, client, bucketName)
|
||||
|
||||
// Create test data for multipart upload (15MB = 3 parts of 5MB each)
|
||||
totalSize := 15 * 1024 * 1024
|
||||
testData := generateRandomData(totalSize)
|
||||
objectKey := "multipart-file.bin"
|
||||
|
||||
expectedPartCount := (totalSize + multipartSize - 1) / multipartSize // ceiling division
|
||||
t.Logf("Performing multipart upload (%d bytes, %d parts)...",
|
||||
totalSize, expectedPartCount)
|
||||
|
||||
// Initiate multipart upload
|
||||
createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to create multipart upload")
|
||||
|
||||
uploadId := createResp.UploadId
|
||||
var completedParts []types.CompletedPart
|
||||
partNumber := int32(1)
|
||||
|
||||
// Upload parts
|
||||
for offset := 0; offset < totalSize; offset += multipartSize {
|
||||
end := offset + multipartSize
|
||||
if end > totalSize {
|
||||
end = totalSize
|
||||
}
|
||||
partData := testData[offset:end]
|
||||
|
||||
uploadResp, err := client.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadId,
|
||||
PartNumber: aws.Int32(partNumber),
|
||||
Body: bytes.NewReader(partData),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload part %d", partNumber)
|
||||
|
||||
completedParts = append(completedParts, types.CompletedPart{
|
||||
ETag: uploadResp.ETag,
|
||||
PartNumber: aws.Int32(partNumber),
|
||||
})
|
||||
partNumber++
|
||||
}
|
||||
|
||||
// Complete multipart upload
|
||||
completeResp, err := client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: uploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: completedParts,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to complete multipart upload")
|
||||
|
||||
// Verify ETag format - SHOULD be composite for multipart
|
||||
etag := aws.ToString(completeResp.ETag)
|
||||
t.Logf("Multipart upload ETag: %s", etag)
|
||||
|
||||
assert.True(t, isCompositeETag(etag),
|
||||
"Multipart upload ETag SHOULD be composite format (md5-count), got: %s", etag)
|
||||
assert.True(t, strings.Contains(cleanETag(etag), "-"),
|
||||
"Multipart ETag should contain hyphen, got: %s", etag)
|
||||
|
||||
// Verify the part count in the ETag matches
|
||||
parts := strings.Split(cleanETag(etag), "-")
|
||||
require.Len(t, parts, 2, "Composite ETag should have format 'hash-count'")
|
||||
assert.Equal(t, fmt.Sprintf("%d", len(completedParts)), parts[1],
|
||||
"Part count in ETag should match number of parts uploaded")
|
||||
}
|
||||
|
||||
// TestPutObjectETagConsistency verifies ETag consistency between PUT and GET
|
||||
func TestPutObjectETagConsistency(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := getS3Client(t)
|
||||
|
||||
bucketName := getNewBucketName()
|
||||
err := createTestBucket(ctx, client, bucketName)
|
||||
require.NoError(t, err, "Failed to create test bucket")
|
||||
defer cleanupTestBucket(ctx, client, bucketName)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
size int
|
||||
}{
|
||||
{"tiny", 100},
|
||||
{"small", smallFileSize},
|
||||
{"medium", mediumFileSize},
|
||||
{"large", largeFileSize},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
testData := generateRandomData(tc.size)
|
||||
objectKey := fmt.Sprintf("consistency-test-%s.bin", tc.name)
|
||||
|
||||
// PUT object
|
||||
putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(testData),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
putETag := aws.ToString(putResp.ETag)
|
||||
|
||||
// HEAD object
|
||||
headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
headETag := aws.ToString(headResp.ETag)
|
||||
|
||||
// GET object
|
||||
getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
getETag := aws.ToString(getResp.ETag)
|
||||
getResp.Body.Close()
|
||||
|
||||
// All ETags should match
|
||||
t.Logf("%s (%d bytes): PUT=%s, HEAD=%s, GET=%s",
|
||||
tc.name, tc.size, putETag, headETag, getETag)
|
||||
|
||||
assert.Equal(t, putETag, headETag,
|
||||
"PUT and HEAD ETags should match")
|
||||
assert.Equal(t, putETag, getETag,
|
||||
"PUT and GET ETags should match")
|
||||
|
||||
// All should be pure MD5 (not composite) for regular PutObject
|
||||
assert.True(t, isPureMD5ETag(putETag),
|
||||
"PutObject ETag should be pure MD5, got: %s", putETag)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestETagHexValidation simulates the AWS SDK v2 validation that caused issue #7768
|
||||
func TestETagHexValidation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := getS3Client(t)
|
||||
|
||||
bucketName := getNewBucketName()
|
||||
err := createTestBucket(ctx, client, bucketName)
|
||||
require.NoError(t, err, "Failed to create test bucket")
|
||||
defer cleanupTestBucket(ctx, client, bucketName)
|
||||
|
||||
// Test with a file large enough to trigger auto-chunking
|
||||
testData := generateRandomData(largeFileSize)
|
||||
objectKey := "hex-validation-test.bin"
|
||||
|
||||
putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(testData),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
etag := cleanETag(aws.ToString(putResp.ETag))
|
||||
|
||||
// Simulate AWS SDK v2's hex validation (Base16Codec.decode)
|
||||
// This is what fails in issue #7768 when ETag contains '-'
|
||||
t.Logf("Validating ETag as hex: %s", etag)
|
||||
|
||||
_, err = hex.DecodeString(etag)
|
||||
assert.NoError(t, err,
|
||||
"ETag should be valid hexadecimal (AWS SDK v2 validation). "+
|
||||
"Got ETag: %s. If this fails with 'invalid byte', the ETag contains non-hex chars like '-'",
|
||||
etag)
|
||||
}
|
||||
|
||||
// TestMultipleLargeFileUploads verifies ETag format across multiple large uploads
|
||||
func TestMultipleLargeFileUploads(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := getS3Client(t)
|
||||
|
||||
bucketName := getNewBucketName()
|
||||
err := createTestBucket(ctx, client, bucketName)
|
||||
require.NoError(t, err, "Failed to create test bucket")
|
||||
defer cleanupTestBucket(ctx, client, bucketName)
|
||||
|
||||
numFiles := 5
|
||||
for i := 0; i < numFiles; i++ {
|
||||
testData := generateRandomData(largeFileSize)
|
||||
expectedMD5 := calculateMD5(testData)
|
||||
objectKey := fmt.Sprintf("large-file-%d.bin", i)
|
||||
|
||||
putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(testData),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload file %d", i)
|
||||
|
||||
etag := aws.ToString(putResp.ETag)
|
||||
t.Logf("File %d ETag: %s (expected MD5: %s)", i, etag, expectedMD5)
|
||||
|
||||
assert.True(t, isPureMD5ETag(etag),
|
||||
"File %d ETag should be pure MD5, got: %s", i, etag)
|
||||
assert.Equal(t, expectedMD5, cleanETag(etag),
|
||||
"File %d ETag should match MD5", i)
|
||||
|
||||
// Validate as hex (AWS SDK v2 check)
|
||||
_, err = hex.DecodeString(cleanETag(etag))
|
||||
assert.NoError(t, err, "File %d ETag should be valid hex", i)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
{
|
||||
"endpoint": "http://127.0.0.1:8333",
|
||||
"access_key": "some_access_key1",
|
||||
"secret_key": "some_secret_key1",
|
||||
"region": "us-east-1",
|
||||
"bucket_prefix": "test-etag-",
|
||||
"notes": {
|
||||
"description": "S3 ETag format integration tests",
|
||||
"issue": "https://github.com/seaweedfs/seaweedfs/issues/7768",
|
||||
"auto_chunk_size": "8MB - files larger than this trigger auto-chunking",
|
||||
"test_sizes": {
|
||||
"small": "1KB - single chunk",
|
||||
"medium": "256KB - at inline threshold",
|
||||
"large": "10MB - 2 internal chunks",
|
||||
"xl": "25MB - 4 internal chunks"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
# S3 Filer Group Test Makefile
|
||||
# This Makefile provides targets for running S3 filer group integration tests
|
||||
# These tests verify that S3 bucket operations work correctly when a filer group is configured.
|
||||
|
||||
.PHONY: help build-weed check-deps start-server stop-server test test-with-server clean logs health-check
|
||||
|
||||
# Configuration
|
||||
WEED_BINARY := ../../../weed/weed_binary
|
||||
S3_PORT := 8333
|
||||
MASTER_PORT := 9333
|
||||
MASTER_GRPC_PORT := 19333
|
||||
VOLUME_PORT := 8080
|
||||
FILER_PORT := 8888
|
||||
TEST_TIMEOUT := 10m
|
||||
TEST_PATTERN := Test
|
||||
|
||||
# Filer group configuration
|
||||
FILER_GROUP := testgroup
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@echo "S3 Filer Group Test Makefile"
|
||||
@echo ""
|
||||
@echo "Available targets:"
|
||||
@echo " help - Show this help message"
|
||||
@echo " build-weed - Build the SeaweedFS binary"
|
||||
@echo " check-deps - Check dependencies and build binary if needed"
|
||||
@echo " start-server - Start SeaweedFS server with filer group"
|
||||
@echo " stop-server - Stop SeaweedFS server"
|
||||
@echo " test - Run filer group tests (server must be running)"
|
||||
@echo " test-with-server - Start server, run tests, stop server"
|
||||
@echo " logs - Show server logs"
|
||||
@echo " clean - Clean up test artifacts and stop server"
|
||||
@echo " health-check - Check if server is accessible"
|
||||
@echo ""
|
||||
@echo "Configuration:"
|
||||
@echo " S3_PORT=${S3_PORT}"
|
||||
@echo " MASTER_GRPC_PORT=${MASTER_GRPC_PORT}"
|
||||
@echo " FILER_GROUP=${FILER_GROUP}"
|
||||
@echo " TEST_TIMEOUT=${TEST_TIMEOUT}"
|
||||
|
||||
# Build the SeaweedFS binary
|
||||
build-weed:
|
||||
@echo "Building SeaweedFS binary..."
|
||||
@cd ../../../weed && go build -o weed_binary .
|
||||
@chmod +x $(WEED_BINARY)
|
||||
@echo "✅ SeaweedFS binary built at $(WEED_BINARY)"
|
||||
|
||||
check-deps: build-weed
|
||||
@echo "Checking dependencies..."
|
||||
@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
|
||||
@echo "Go version: $$(go version)"
|
||||
@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
|
||||
@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1)
|
||||
@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1)
|
||||
@echo "✅ All dependencies are available"
|
||||
|
||||
# Start SeaweedFS server with filer group configured
|
||||
start-server: check-deps
|
||||
@echo "Starting SeaweedFS server with filer group: $(FILER_GROUP)..."
|
||||
@rm -f weed-server.pid
|
||||
@mkdir -p ./test-volume-data
|
||||
@if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \
|
||||
echo "⚠️ Port $(S3_PORT) is already in use"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "Launching SeaweedFS server with filer group $(FILER_GROUP)..."
|
||||
@$(WEED_BINARY) server \
|
||||
-debug \
|
||||
-s3 \
|
||||
-s3.port=$(S3_PORT) \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
-s3.config=../../../docker/compose/s3.json \
|
||||
-filer \
|
||||
-filer.maxMB=64 \
|
||||
-filer.filerGroup=$(FILER_GROUP) \
|
||||
-master.volumeSizeLimitMB=50 \
|
||||
-volume.max=100 \
|
||||
-dir=./test-volume-data \
|
||||
-volume.preStopSeconds=1 \
|
||||
-metricsPort=9325 \
|
||||
> weed-test.log 2>&1 & echo $$! > weed-server.pid
|
||||
@echo "Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
|
||||
@echo "Waiting for server to start (up to 90 seconds)..."
|
||||
@for i in $$(seq 1 90); do \
|
||||
if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
|
||||
echo "✅ SeaweedFS server started successfully on port $(S3_PORT) with filer group $(FILER_GROUP)"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
if [ $$i -eq 30 ]; then \
|
||||
echo "⚠️ Server taking longer than expected (30s), checking logs..."; \
|
||||
if [ -f weed-test.log ]; then \
|
||||
tail -20 weed-test.log; \
|
||||
fi; \
|
||||
fi; \
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "❌ Server failed to start within 90 seconds"; \
|
||||
if [ -f weed-test.log ]; then \
|
||||
cat weed-test.log; \
|
||||
fi; \
|
||||
exit 1
|
||||
|
||||
# Stop SeaweedFS server
|
||||
stop-server:
|
||||
@echo "Stopping SeaweedFS server..."
|
||||
@if [ -f weed-server.pid ]; then \
|
||||
SERVER_PID=$$(cat weed-server.pid); \
|
||||
echo "Killing server PID $$SERVER_PID"; \
|
||||
if ps -p $$SERVER_PID >/dev/null 2>&1; then \
|
||||
kill -TERM $$SERVER_PID 2>/dev/null || true; \
|
||||
sleep 2; \
|
||||
if ps -p $$SERVER_PID >/dev/null 2>&1; then \
|
||||
echo "Process still running, sending KILL signal..."; \
|
||||
kill -KILL $$SERVER_PID 2>/dev/null || true; \
|
||||
sleep 1; \
|
||||
fi; \
|
||||
else \
|
||||
echo "Process $$SERVER_PID not found (already stopped)"; \
|
||||
fi; \
|
||||
rm -f weed-server.pid; \
|
||||
else \
|
||||
echo "No PID file found"; \
|
||||
fi
|
||||
@echo "✅ SeaweedFS server stopped"
|
||||
|
||||
# Show server logs
|
||||
logs:
|
||||
@if test -f weed-test.log; then \
|
||||
echo "=== SeaweedFS Server Logs ==="; \
|
||||
tail -f weed-test.log; \
|
||||
else \
|
||||
echo "No log file found. Server may not be running."; \
|
||||
fi
|
||||
|
||||
# Run filer group tests (assumes server is already running)
|
||||
test: check-deps
|
||||
@echo "Running filer group S3 tests..."
|
||||
@FILER_GROUP=$(FILER_GROUP) S3_ENDPOINT=http://localhost:$(S3_PORT) MASTER_ADDRESS=localhost:$(MASTER_GRPC_PORT) \
|
||||
go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
|
||||
@echo "✅ Filer group tests completed"
|
||||
|
||||
# Run tests with automatic server management
|
||||
test-with-server: start-server
|
||||
@echo "Server started successfully, running filer group tests..."
|
||||
@echo "Test pattern: $(TEST_PATTERN)"
|
||||
@echo "Test timeout: $(TEST_TIMEOUT)"
|
||||
@trap "$(MAKE) stop-server" EXIT; \
|
||||
$(MAKE) test || (echo "❌ Tests failed, showing server logs:" && echo "=== Last 50 lines of server logs ===" && tail -50 weed-test.log && echo "=== End of server logs ===" && exit 1)
|
||||
@$(MAKE) stop-server
|
||||
@echo "✅ Tests completed and server stopped"
|
||||
|
||||
# Clean up test artifacts
|
||||
clean:
|
||||
@echo "Cleaning up test artifacts..."
|
||||
@$(MAKE) stop-server
|
||||
@rm -f weed-test*.log weed-server.pid
|
||||
@rm -rf test-volume-data/
|
||||
@go clean -testcache
|
||||
@echo "✅ Cleanup completed"
|
||||
|
||||
# Quick health check
|
||||
health-check:
|
||||
@echo "Running health check..."
|
||||
@curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1 && echo "✅ S3 API is accessible" || echo "❌ S3 API is not accessible"
|
||||
@curl -s http://localhost:9325/metrics >/dev/null 2>&1 && echo "✅ Metrics endpoint is accessible" || echo "❌ Metrics endpoint is not accessible"
|
||||
@@ -1,99 +0,0 @@
|
||||
# Filer Group S3 Tests
|
||||
|
||||
These tests verify that S3 bucket operations work correctly when a filer group is configured.
|
||||
|
||||
## Background
|
||||
|
||||
When SeaweedFS is configured with a filer group (via `-filer.group` option), collections are named with the filer group prefix:
|
||||
|
||||
```text
|
||||
Collection name = {filerGroup}_{bucketName}
|
||||
```
|
||||
|
||||
For example, with filer group `mygroup` and bucket `mybucket`, the collection will be named `mygroup_mybucket`.
|
||||
|
||||
## Issue Being Tested
|
||||
|
||||
This test suite was created to verify the fix for a bug where:
|
||||
- The admin UI was using just the bucket name when deleting collections
|
||||
- This caused collection deletion to fail when a filer group was configured
|
||||
- After bucket deletion via admin UI, the collection data would be orphaned
|
||||
|
||||
## Running the Tests
|
||||
|
||||
### Prerequisites
|
||||
|
||||
1. SeaweedFS servers must be running with a filer group configured
|
||||
2. The S3 gateway must be accessible
|
||||
3. Master server must be accessible for collection verification
|
||||
|
||||
### Quick Start
|
||||
|
||||
```bash
|
||||
# Set environment variables
|
||||
export FILER_GROUP=testgroup
|
||||
export S3_ENDPOINT=http://localhost:8333
|
||||
export MASTER_ADDRESS=localhost:9333
|
||||
|
||||
# Run tests
|
||||
go test -v ./...
|
||||
```
|
||||
|
||||
### Using the Makefile
|
||||
|
||||
```bash
|
||||
# Start test servers with filer group configured
|
||||
make start-servers FILER_GROUP=testgroup
|
||||
|
||||
# Run tests
|
||||
make test
|
||||
|
||||
# Stop servers
|
||||
make stop-servers
|
||||
|
||||
# Or run full test cycle
|
||||
make full-test
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
Tests can be configured via:
|
||||
|
||||
1. Environment variables:
|
||||
- `FILER_GROUP`: The filer group name (required for tests to run)
|
||||
- `S3_ENDPOINT`: S3 API endpoint (default: `http://localhost:8333`)
|
||||
- `MASTER_ADDRESS`: Master server address (default: localhost:9333)
|
||||
|
||||
2. `test_config.json` file
|
||||
|
||||
## Test Cases
|
||||
|
||||
### TestFilerGroupCollectionNaming
|
||||
Verifies that when a bucket is created and objects are uploaded:
|
||||
1. The collection is created with the correct filer group prefix
|
||||
2. Bucket deletion removes the correctly-named collection
|
||||
|
||||
### TestBucketDeletionWithFilerGroup
|
||||
Specifically tests that bucket deletion via S3 API correctly deletes
|
||||
the collection when filer group is configured.
|
||||
|
||||
### TestMultipleBucketsWithFilerGroup
|
||||
Tests creating and deleting multiple buckets to ensure the filer group
|
||||
prefix is correctly applied and removed for all buckets.
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
With filer group `testgroup`:
|
||||
|
||||
| Bucket Name | Expected Collection Name |
|
||||
|-------------|-------------------------|
|
||||
| mybucket | testgroup_mybucket |
|
||||
| test-123 | testgroup_test-123 |
|
||||
|
||||
Without filer group:
|
||||
|
||||
| Bucket Name | Expected Collection Name |
|
||||
|-------------|-------------------------|
|
||||
| mybucket | mybucket |
|
||||
| test-123 | test-123 |
|
||||
|
||||
@@ -1,332 +0,0 @@
|
||||
package filer_group
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
// TestConfig holds configuration for filer group S3 tests
|
||||
type TestConfig struct {
|
||||
S3Endpoint string `json:"s3_endpoint"`
|
||||
MasterAddress string `json:"master_address"`
|
||||
AccessKey string `json:"access_key"`
|
||||
SecretKey string `json:"secret_key"`
|
||||
Region string `json:"region"`
|
||||
FilerGroup string `json:"filer_group"`
|
||||
}
|
||||
|
||||
var testConfig = &TestConfig{
|
||||
S3Endpoint: "http://localhost:8333",
|
||||
MasterAddress: "localhost:19333", // gRPC port = 10000 + master HTTP port (9333)
|
||||
AccessKey: "some_access_key1",
|
||||
SecretKey: "some_secret_key1",
|
||||
Region: "us-east-1",
|
||||
FilerGroup: "testgroup", // Expected filer group for these tests
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Load config from file if exists
|
||||
if data, err := os.ReadFile("test_config.json"); err == nil {
|
||||
if err := json.Unmarshal(data, testConfig); err != nil {
|
||||
// Log but don't fail - env vars can still override
|
||||
fmt.Fprintf(os.Stderr, "Warning: failed to parse test_config.json: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Override from environment variables
|
||||
if v := os.Getenv("S3_ENDPOINT"); v != "" {
|
||||
testConfig.S3Endpoint = v
|
||||
}
|
||||
if v := os.Getenv("MASTER_ADDRESS"); v != "" {
|
||||
testConfig.MasterAddress = v
|
||||
}
|
||||
if v := os.Getenv("FILER_GROUP"); v != "" {
|
||||
testConfig.FilerGroup = v
|
||||
}
|
||||
}
|
||||
|
||||
func getS3Client(t *testing.T) *s3.Client {
|
||||
cfg, err := config.LoadDefaultConfig(context.TODO(),
|
||||
config.WithRegion(testConfig.Region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
|
||||
testConfig.AccessKey,
|
||||
testConfig.SecretKey,
|
||||
"",
|
||||
)),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
return s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(testConfig.S3Endpoint)
|
||||
o.UsePathStyle = true
|
||||
})
|
||||
}
|
||||
|
||||
func getMasterClient(t *testing.T) master_pb.SeaweedClient {
|
||||
conn, err := grpc.NewClient(testConfig.MasterAddress, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { conn.Close() })
|
||||
return master_pb.NewSeaweedClient(conn)
|
||||
}
|
||||
|
||||
func getNewBucketName() string {
|
||||
return fmt.Sprintf("filergroup-test-%d", time.Now().UnixNano())
|
||||
}
|
||||
|
||||
// getExpectedCollectionName returns the expected collection name for a bucket
|
||||
// When filer group is configured, collections are named: {filerGroup}_{bucketName}
|
||||
func getExpectedCollectionName(bucketName string) string {
|
||||
if testConfig.FilerGroup != "" {
|
||||
return fmt.Sprintf("%s_%s", testConfig.FilerGroup, bucketName)
|
||||
}
|
||||
return bucketName
|
||||
}
|
||||
|
||||
// listAllCollections returns a list of all collection names from the master
|
||||
func listAllCollections(t *testing.T, masterClient master_pb.SeaweedClient) []string {
|
||||
collectionResp, err := masterClient.CollectionList(context.Background(), &master_pb.CollectionListRequest{
|
||||
IncludeNormalVolumes: true,
|
||||
IncludeEcVolumes: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("Warning: failed to list collections: %v", err)
|
||||
return nil
|
||||
}
|
||||
var names []string
|
||||
for _, c := range collectionResp.Collections {
|
||||
names = append(names, c.Name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// collectionExists checks if a collection exists in the master
|
||||
func collectionExists(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) bool {
|
||||
for _, name := range listAllCollections(t, masterClient) {
|
||||
if name == collectionName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// waitForCollectionExists waits for a collection to exist using polling
|
||||
func waitForCollectionExists(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) {
|
||||
var lastCollections []string
|
||||
success := assert.Eventually(t, func() bool {
|
||||
lastCollections = listAllCollections(t, masterClient)
|
||||
for _, name := range lastCollections {
|
||||
if name == collectionName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}, 10*time.Second, 200*time.Millisecond)
|
||||
if !success {
|
||||
t.Fatalf("collection %s should be created; existing collections: %v", collectionName, lastCollections)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForCollectionDeleted waits for a collection to be deleted using polling
|
||||
func waitForCollectionDeleted(t *testing.T, masterClient master_pb.SeaweedClient, collectionName string) {
|
||||
require.Eventually(t, func() bool {
|
||||
return !collectionExists(t, masterClient, collectionName)
|
||||
}, 10*time.Second, 200*time.Millisecond, "collection %s should be deleted", collectionName)
|
||||
}
|
||||
|
||||
// TestFilerGroupCollectionNaming verifies that when a filer group is configured,
|
||||
// collections are created with the correct prefix ({filerGroup}_{bucketName})
|
||||
func TestFilerGroupCollectionNaming(t *testing.T) {
|
||||
if testConfig.FilerGroup == "" {
|
||||
t.Skip("Skipping test: FILER_GROUP not configured. Set FILER_GROUP environment variable or configure in test_config.json")
|
||||
}
|
||||
|
||||
s3Client := getS3Client(t)
|
||||
masterClient := getMasterClient(t)
|
||||
bucketName := getNewBucketName()
|
||||
expectedCollection := getExpectedCollectionName(bucketName)
|
||||
|
||||
t.Logf("Testing with filer group: %s", testConfig.FilerGroup)
|
||||
t.Logf("Bucket name: %s", bucketName)
|
||||
t.Logf("Expected collection name: %s", expectedCollection)
|
||||
|
||||
// Create bucket
|
||||
_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err, "CreateBucket should succeed")
|
||||
|
||||
// Upload an object to trigger collection creation
|
||||
_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("test-object"),
|
||||
Body: strings.NewReader("test content"),
|
||||
})
|
||||
require.NoError(t, err, "PutObject should succeed")
|
||||
|
||||
// Wait for collection to be visible using polling
|
||||
waitForCollectionExists(t, masterClient, expectedCollection)
|
||||
|
||||
// Verify collection exists with correct name
|
||||
require.True(t, collectionExists(t, masterClient, expectedCollection),
|
||||
"Collection %s should exist (filer group prefix applied)", expectedCollection)
|
||||
|
||||
// Cleanup: delete object and bucket
|
||||
_, err = s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("test-object"),
|
||||
})
|
||||
require.NoError(t, err, "DeleteObject should succeed")
|
||||
|
||||
_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err, "DeleteBucket should succeed")
|
||||
|
||||
// Wait for collection to be deleted using polling
|
||||
waitForCollectionDeleted(t, masterClient, expectedCollection)
|
||||
|
||||
t.Log("SUCCESS: Collection naming with filer group works correctly")
|
||||
}
|
||||
|
||||
// TestBucketDeletionWithFilerGroup verifies that bucket deletion correctly
|
||||
// deletes the collection when filer group is configured
|
||||
func TestBucketDeletionWithFilerGroup(t *testing.T) {
|
||||
if testConfig.FilerGroup == "" {
|
||||
t.Skip("Skipping test: FILER_GROUP not configured")
|
||||
}
|
||||
|
||||
s3Client := getS3Client(t)
|
||||
masterClient := getMasterClient(t)
|
||||
bucketName := getNewBucketName()
|
||||
expectedCollection := getExpectedCollectionName(bucketName)
|
||||
|
||||
// Create bucket and add an object
|
||||
_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("test-object"),
|
||||
Body: strings.NewReader("test content"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Wait for collection to be created using polling
|
||||
waitForCollectionExists(t, masterClient, expectedCollection)
|
||||
|
||||
// Verify collection exists before deletion
|
||||
require.True(t, collectionExists(t, masterClient, expectedCollection),
|
||||
"Collection should exist before bucket deletion")
|
||||
|
||||
// Delete object first
|
||||
_, err = s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("test-object"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Delete bucket
|
||||
_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err, "DeleteBucket should succeed")
|
||||
|
||||
// Wait for collection to be deleted using polling
|
||||
waitForCollectionDeleted(t, masterClient, expectedCollection)
|
||||
|
||||
// Verify collection was deleted
|
||||
require.False(t, collectionExists(t, masterClient, expectedCollection),
|
||||
"Collection %s should be deleted after bucket deletion", expectedCollection)
|
||||
|
||||
t.Log("SUCCESS: Bucket deletion with filer group correctly deletes collection")
|
||||
}
|
||||
|
||||
// TestMultipleBucketsWithFilerGroup tests creating and deleting multiple buckets
|
||||
func TestMultipleBucketsWithFilerGroup(t *testing.T) {
|
||||
if testConfig.FilerGroup == "" {
|
||||
t.Skip("Skipping test: FILER_GROUP not configured")
|
||||
}
|
||||
|
||||
s3Client := getS3Client(t)
|
||||
masterClient := getMasterClient(t)
|
||||
|
||||
buckets := make([]string, 3)
|
||||
for i := 0; i < 3; i++ {
|
||||
buckets[i] = getNewBucketName()
|
||||
}
|
||||
|
||||
// Create all buckets and add objects
|
||||
for _, bucket := range buckets {
|
||||
_, err := s3Client.CreateBucket(context.Background(), &s3.CreateBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = s3Client.PutObject(context.Background(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("test-object"),
|
||||
Body: strings.NewReader("test content"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Wait for all collections to be created using polling
|
||||
for _, bucket := range buckets {
|
||||
expectedCollection := getExpectedCollectionName(bucket)
|
||||
waitForCollectionExists(t, masterClient, expectedCollection)
|
||||
}
|
||||
|
||||
// Verify all collections exist with correct naming
|
||||
for _, bucket := range buckets {
|
||||
expectedCollection := getExpectedCollectionName(bucket)
|
||||
require.True(t, collectionExists(t, masterClient, expectedCollection),
|
||||
"Collection %s should exist for bucket %s", expectedCollection, bucket)
|
||||
}
|
||||
|
||||
// Delete all buckets
|
||||
for _, bucket := range buckets {
|
||||
_, err := s3Client.DeleteObject(context.Background(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String("test-object"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = s3Client.DeleteBucket(context.Background(), &s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucket),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Wait for all collections to be deleted using polling
|
||||
for _, bucket := range buckets {
|
||||
expectedCollection := getExpectedCollectionName(bucket)
|
||||
waitForCollectionDeleted(t, masterClient, expectedCollection)
|
||||
}
|
||||
|
||||
// Verify all collections are deleted
|
||||
for _, bucket := range buckets {
|
||||
expectedCollection := getExpectedCollectionName(bucket)
|
||||
require.False(t, collectionExists(t, masterClient, expectedCollection),
|
||||
"Collection %s should be deleted for bucket %s", expectedCollection, bucket)
|
||||
}
|
||||
|
||||
t.Log("SUCCESS: Multiple bucket operations with filer group work correctly")
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
{
|
||||
"s3_endpoint": "http://localhost:8333",
|
||||
"master_address": "localhost:9333",
|
||||
"access_key": "some_access_key1",
|
||||
"secret_key": "some_secret_key1",
|
||||
"region": "us-east-1",
|
||||
"filer_group": "testgroup"
|
||||
}
|
||||
@@ -558,14 +558,13 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {
|
||||
adminClient, err := framework.CreateS3ClientWithJWT("admin-user", "TestAdminRole")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Use unique bucket name to avoid conflicts with other tests
|
||||
bucketName := framework.GenerateUniqueBucketName("test-iam-presigned")
|
||||
err = framework.CreateBucket(adminClient, bucketName)
|
||||
// Use static bucket name but with cleanup to handle conflicts
|
||||
err = framework.CreateBucketWithCleanup(adminClient, testBucket)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Put test object
|
||||
_, err = adminClient.PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Bucket: aws.String(testBucket),
|
||||
Key: aws.String(testObjectKey),
|
||||
Body: strings.NewReader(testObjectData),
|
||||
})
|
||||
@@ -587,7 +586,7 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {
|
||||
|
||||
// Test direct object access with JWT Bearer token (recommended approach)
|
||||
_, err := adminClient.GetObject(&s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Bucket: aws.String(testBucket),
|
||||
Key: aws.String(testObjectKey),
|
||||
})
|
||||
require.NoError(t, err, "Direct object access with JWT Bearer token works correctly")
|
||||
@@ -598,13 +597,13 @@ func TestS3IAMPresignedURLIntegration(t *testing.T) {
|
||||
|
||||
// Cleanup
|
||||
_, err = adminClient.DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Bucket: aws.String(testBucket),
|
||||
Key: aws.String(testObjectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = adminClient.DeleteBucket(&s3.DeleteBucketInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Bucket: aws.String(testBucket),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
"sts": {
|
||||
"tokenDuration": "15m",
|
||||
"issuer": "seaweedfs-sts",
|
||||
"signingKey": "dGVzdC1zaWduaW5nLWtleS0zMi1jaGFyYWN0ZXJzLWxvbmc="
|
||||
"signingKey": "test-sts-signing-key-for-integration-tests"
|
||||
},
|
||||
"policy": {
|
||||
"defaultEffect": "Deny"
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
# Remote Storage Cache Integration Tests
|
||||
# Tests the remote object caching functionality with singleflight deduplication
|
||||
# Uses two SeaweedFS instances: primary (with caching) and secondary (as remote storage)
|
||||
|
||||
.PHONY: all help build-weed check-deps start-remote stop-remote start-primary stop-primary \
|
||||
setup-remote test test-with-server clean logs logs-primary logs-remote health
|
||||
|
||||
all: test-with-server
|
||||
|
||||
# Configuration
|
||||
WEED_BINARY := ../../../weed/weed_binary
|
||||
|
||||
# Primary SeaweedFS (the one being tested - has remote caching)
|
||||
PRIMARY_S3_PORT := 8333
|
||||
PRIMARY_FILER_PORT := 8888
|
||||
PRIMARY_MASTER_PORT := 9333
|
||||
PRIMARY_VOLUME_PORT := 8080
|
||||
PRIMARY_METRICS_PORT := 9324
|
||||
PRIMARY_DIR := ./test-primary-data
|
||||
|
||||
# Secondary SeaweedFS (acts as "remote" S3 storage)
|
||||
REMOTE_S3_PORT := 8334
|
||||
REMOTE_FILER_PORT := 8889
|
||||
REMOTE_MASTER_PORT := 9334
|
||||
REMOTE_VOLUME_PORT := 8081
|
||||
REMOTE_METRICS_PORT := 9325
|
||||
REMOTE_DIR := ./test-remote-data
|
||||
|
||||
# Test configuration
|
||||
TEST_TIMEOUT := 10m
|
||||
TEST_PATTERN := TestRemoteCache
|
||||
|
||||
# Buckets
|
||||
REMOTE_BUCKET := remotesourcebucket
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@echo "Remote Storage Cache Integration Tests"
|
||||
@echo ""
|
||||
@echo "Uses two SeaweedFS instances:"
|
||||
@echo " - Primary (port $(PRIMARY_S3_PORT)): Being tested, has remote caching"
|
||||
@echo " - Remote (port $(REMOTE_S3_PORT)): Acts as remote S3 storage"
|
||||
@echo ""
|
||||
@echo "Available targets:"
|
||||
@echo " help - Show this help message"
|
||||
@echo " build-weed - Build the SeaweedFS binary"
|
||||
@echo " check-deps - Check dependencies"
|
||||
@echo " start-remote - Start remote SeaweedFS (secondary)"
|
||||
@echo " stop-remote - Stop remote SeaweedFS"
|
||||
@echo " start-primary - Start primary SeaweedFS"
|
||||
@echo " stop-primary - Stop primary SeaweedFS"
|
||||
@echo " setup-remote - Configure remote storage mount"
|
||||
@echo " test - Run tests (assumes servers are running)"
|
||||
@echo " test-with-server - Start servers, run tests, stop servers"
|
||||
@echo " clean - Clean up all resources"
|
||||
@echo " logs - Show server logs"
|
||||
|
||||
# Build the SeaweedFS binary
|
||||
build-weed:
|
||||
@echo "Building SeaweedFS binary..."
|
||||
@cd ../../../weed && go build -o weed_binary .
|
||||
@chmod +x $(WEED_BINARY)
|
||||
@echo "SeaweedFS binary built"
|
||||
|
||||
check-deps: build-weed
|
||||
@echo "Checking dependencies..."
|
||||
@command -v go >/dev/null 2>&1 || (echo "Go is required" && exit 1)
|
||||
@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found" && exit 1)
|
||||
@echo "All dependencies available"
|
||||
|
||||
# Start remote SeaweedFS (acts as the "remote" S3 storage)
|
||||
start-remote: check-deps
|
||||
@echo "Starting remote SeaweedFS (secondary instance)..."
|
||||
@rm -f remote-server.pid
|
||||
@mkdir -p $(REMOTE_DIR)
|
||||
@$(WEED_BINARY) server \
|
||||
-s3 \
|
||||
-s3.port=$(REMOTE_S3_PORT) \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
-filer \
|
||||
-filer.port=$(REMOTE_FILER_PORT) \
|
||||
-master.port=$(REMOTE_MASTER_PORT) \
|
||||
-volume.port=$(REMOTE_VOLUME_PORT) \
|
||||
-master.volumeSizeLimitMB=50 \
|
||||
-volume.max=100 \
|
||||
-dir=$(REMOTE_DIR) \
|
||||
-volume.preStopSeconds=1 \
|
||||
-metricsPort=$(REMOTE_METRICS_PORT) \
|
||||
> remote-weed.log 2>&1 & echo $$! > remote-server.pid
|
||||
@echo "Waiting for remote SeaweedFS to start..."
|
||||
@for i in $$(seq 1 60); do \
|
||||
if curl -s http://localhost:$(REMOTE_S3_PORT) >/dev/null 2>&1; then \
|
||||
echo "Remote SeaweedFS started on port $(REMOTE_S3_PORT)"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "ERROR: Remote SeaweedFS failed to start"; \
|
||||
cat remote-weed.log; \
|
||||
exit 1
|
||||
|
||||
stop-remote:
|
||||
@echo "Stopping remote SeaweedFS..."
|
||||
@if [ -f remote-server.pid ]; then \
|
||||
kill -TERM $$(cat remote-server.pid) 2>/dev/null || true; \
|
||||
sleep 2; \
|
||||
kill -KILL $$(cat remote-server.pid) 2>/dev/null || true; \
|
||||
rm -f remote-server.pid; \
|
||||
fi
|
||||
@echo "Remote SeaweedFS stopped"
|
||||
|
||||
# Start primary SeaweedFS (the one being tested)
|
||||
start-primary: check-deps
|
||||
@echo "Starting primary SeaweedFS..."
|
||||
@rm -f primary-server.pid
|
||||
@mkdir -p $(PRIMARY_DIR)
|
||||
@$(WEED_BINARY) server \
|
||||
-s3 \
|
||||
-s3.port=$(PRIMARY_S3_PORT) \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
-filer \
|
||||
-filer.port=$(PRIMARY_FILER_PORT) \
|
||||
-master.port=$(PRIMARY_MASTER_PORT) \
|
||||
-volume.port=$(PRIMARY_VOLUME_PORT) \
|
||||
-master.volumeSizeLimitMB=50 \
|
||||
-volume.max=100 \
|
||||
-dir=$(PRIMARY_DIR) \
|
||||
-volume.preStopSeconds=1 \
|
||||
-metricsPort=$(PRIMARY_METRICS_PORT) \
|
||||
> primary-weed.log 2>&1 & echo $$! > primary-server.pid
|
||||
@echo "Waiting for primary SeaweedFS to start..."
|
||||
@for i in $$(seq 1 60); do \
|
||||
if curl -s http://localhost:$(PRIMARY_S3_PORT) >/dev/null 2>&1; then \
|
||||
echo "Primary SeaweedFS started on port $(PRIMARY_S3_PORT)"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "ERROR: Primary SeaweedFS failed to start"; \
|
||||
cat primary-weed.log; \
|
||||
exit 1
|
||||
|
||||
stop-primary:
|
||||
@echo "Stopping primary SeaweedFS..."
|
||||
@if [ -f primary-server.pid ]; then \
|
||||
kill -TERM $$(cat primary-server.pid) 2>/dev/null || true; \
|
||||
sleep 2; \
|
||||
kill -KILL $$(cat primary-server.pid) 2>/dev/null || true; \
|
||||
rm -f primary-server.pid; \
|
||||
fi
|
||||
@echo "Primary SeaweedFS stopped"
|
||||
|
||||
# Create bucket on remote and configure remote storage mount on primary
|
||||
setup-remote:
|
||||
@echo "Creating bucket on remote SeaweedFS..."
|
||||
@curl -s -X PUT "http://localhost:$(REMOTE_S3_PORT)/$(REMOTE_BUCKET)" || echo "Bucket may already exist"
|
||||
@sleep 1
|
||||
@echo "Configuring remote storage on primary..."
|
||||
@printf 'remote.configure -name=seaweedremote -type=s3 -s3.access_key=any -s3.secret_key=any -s3.endpoint=http://localhost:$(REMOTE_S3_PORT) -s3.region=us-east-1\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.configure done"
|
||||
@sleep 2
|
||||
@echo "Mounting remote bucket on primary..."
|
||||
@printf 'remote.mount -dir=/buckets/remotemounted -remote=seaweedremote/$(REMOTE_BUCKET) -nonempty\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.mount done"
|
||||
@sleep 1
|
||||
@echo "Remote storage configured"
|
||||
|
||||
# Run tests
|
||||
test: check-deps
|
||||
@echo "Running remote cache tests..."
|
||||
@go test -v -timeout=$(TEST_TIMEOUT) -run "$(TEST_PATTERN)" .
|
||||
@echo "Tests completed"
|
||||
|
||||
# Full test workflow
|
||||
test-with-server: start-remote start-primary
|
||||
@sleep 3
|
||||
@$(MAKE) setup-remote || (echo "Remote setup failed" && $(MAKE) stop-primary stop-remote && exit 1)
|
||||
@sleep 2
|
||||
@echo "Running remote cache tests..."
|
||||
@$(MAKE) test || (echo "Tests failed" && tail -50 primary-weed.log && $(MAKE) stop-primary stop-remote && exit 1)
|
||||
@$(MAKE) stop-primary stop-remote
|
||||
@echo "All tests passed"
|
||||
|
||||
# Show logs
|
||||
logs:
|
||||
@echo "=== Primary SeaweedFS Logs ==="
|
||||
@if [ -f primary-weed.log ]; then tail -50 primary-weed.log; else echo "No log file"; fi
|
||||
@echo ""
|
||||
@echo "=== Remote SeaweedFS Logs ==="
|
||||
@if [ -f remote-weed.log ]; then tail -50 remote-weed.log; else echo "No log file"; fi
|
||||
|
||||
logs-primary:
|
||||
@if [ -f primary-weed.log ]; then tail -f primary-weed.log; else echo "No log file"; fi
|
||||
|
||||
logs-remote:
|
||||
@if [ -f remote-weed.log ]; then tail -f remote-weed.log; else echo "No log file"; fi
|
||||
|
||||
# Clean up
|
||||
clean:
|
||||
@$(MAKE) stop-primary
|
||||
@$(MAKE) stop-remote
|
||||
@rm -f primary-weed.log remote-weed.log primary-server.pid remote-server.pid
|
||||
@rm -rf $(PRIMARY_DIR) $(REMOTE_DIR)
|
||||
@rm -f remote_cache.test
|
||||
@go clean -testcache
|
||||
@echo "Cleanup completed"
|
||||
|
||||
# Health check
|
||||
health:
|
||||
@echo "Checking server status..."
|
||||
@curl -s http://localhost:$(PRIMARY_S3_PORT) >/dev/null 2>&1 && echo "Primary S3 ($(PRIMARY_S3_PORT)): UP" || echo "Primary S3 ($(PRIMARY_S3_PORT)): DOWN"
|
||||
@curl -s http://localhost:$(REMOTE_S3_PORT) >/dev/null 2>&1 && echo "Remote S3 ($(REMOTE_S3_PORT)): UP" || echo "Remote S3 ($(REMOTE_S3_PORT)): DOWN"
|
||||
@@ -1,157 +0,0 @@
|
||||
# Remote Object Cache Integration Tests
|
||||
|
||||
This directory contains integration tests for the remote object caching feature with singleflight deduplication.
|
||||
|
||||
## Test Flow
|
||||
|
||||
Each test follows this pattern:
|
||||
1. **Write to local** - Upload data to primary SeaweedFS (local storage)
|
||||
2. **Uncache** - Push data to remote storage and remove local chunks
|
||||
3. **Read** - Read data (triggers caching from remote back to local)
|
||||
|
||||
This tests the full remote caching workflow including singleflight deduplication.
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Test Client │
|
||||
│ │
|
||||
│ 1. PUT data to primary SeaweedFS │
|
||||
│ 2. remote.cache.uncache (push to remote, purge local) │
|
||||
│ 3. GET data (triggers caching from remote) │
|
||||
│ 4. Verify singleflight deduplication │
|
||||
└──────────────────────────────────┬──────────────────────────────┘
|
||||
│
|
||||
┌─────────────────┴─────────────────┐
|
||||
▼ ▼
|
||||
┌────────────────────────────────────┐ ┌────────────────────────────────┐
|
||||
│ Primary SeaweedFS │ │ Remote SeaweedFS │
|
||||
│ (port 8333) │ │ (port 8334) │
|
||||
│ │ │ │
|
||||
│ - Being tested │ │ - Acts as "remote" S3 │
|
||||
│ - Has remote storage mounted │──▶│ - Receives uncached data │
|
||||
│ - Caches remote objects │ │ - Serves data for caching │
|
||||
│ - Singleflight deduplication │ │ │
|
||||
└────────────────────────────────────┘ └────────────────────────────────┘
|
||||
```
|
||||
|
||||
## What's Being Tested
|
||||
|
||||
1. **Basic Remote Caching**: Write → Uncache → Read workflow
|
||||
2. **Singleflight Deduplication**: Concurrent reads only trigger ONE caching operation
|
||||
3. **Large Object Caching**: 5MB files cache correctly
|
||||
4. **Range Requests**: Partial reads work with cached objects
|
||||
5. **Not Found Handling**: Proper error for non-existent objects
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Run Full Test Suite (Recommended)
|
||||
|
||||
```bash
|
||||
# Build SeaweedFS, start both servers, run tests, stop servers
|
||||
make test-with-server
|
||||
```
|
||||
|
||||
### Manual Steps
|
||||
|
||||
```bash
|
||||
# 1. Build SeaweedFS binary
|
||||
make build-weed
|
||||
|
||||
# 2. Start remote SeaweedFS (acts as "remote" storage)
|
||||
make start-remote
|
||||
|
||||
# 3. Start primary SeaweedFS (the one being tested)
|
||||
make start-primary
|
||||
|
||||
# 4. Configure remote storage mount
|
||||
make setup-remote
|
||||
|
||||
# 5. Run tests
|
||||
make test
|
||||
|
||||
# 6. Clean up
|
||||
make clean
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
### Primary SeaweedFS (Being Tested)
|
||||
|
||||
| Service | Port |
|
||||
|---------|------|
|
||||
| S3 API | 8333 |
|
||||
| Filer | 8888 |
|
||||
| Master | 9333 |
|
||||
| Volume | 8080 |
|
||||
|
||||
### Remote SeaweedFS (Remote Storage)
|
||||
|
||||
| Service | Port |
|
||||
|---------|------|
|
||||
| S3 API | 8334 |
|
||||
| Filer | 8889 |
|
||||
| Master | 9334 |
|
||||
| Volume | 8081 |
|
||||
|
||||
## Makefile Targets
|
||||
|
||||
```bash
|
||||
make help # Show all available targets
|
||||
make build-weed # Build SeaweedFS binary
|
||||
make start-remote # Start remote SeaweedFS
|
||||
make start-primary # Start primary SeaweedFS
|
||||
make setup-remote # Configure remote storage mount
|
||||
make test # Run tests
|
||||
make test-with-server # Full automated test workflow
|
||||
make logs # Show server logs
|
||||
make health # Check server status
|
||||
make clean # Stop servers and clean up
|
||||
```
|
||||
|
||||
## Test Details
|
||||
|
||||
### TestRemoteCacheBasic
|
||||
Basic workflow test:
|
||||
1. Write object to primary (local)
|
||||
2. Uncache (push to remote, remove local chunks)
|
||||
3. Read (triggers caching from remote)
|
||||
4. Read again (from local cache - should be faster)
|
||||
|
||||
### TestRemoteCacheConcurrent
|
||||
Singleflight deduplication test:
|
||||
1. Write 1MB object
|
||||
2. Uncache to remote
|
||||
3. Launch 10 concurrent reads
|
||||
4. All should succeed with correct data
|
||||
5. Only ONE caching operation should run (singleflight)
|
||||
|
||||
### TestRemoteCacheLargeObject
|
||||
Large file test (5MB) to verify chunked transfer works correctly.
|
||||
|
||||
### TestRemoteCacheRangeRequest
|
||||
Tests HTTP range requests work correctly after caching.
|
||||
|
||||
### TestRemoteCacheNotFound
|
||||
Tests proper error handling for non-existent objects.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### View logs
|
||||
```bash
|
||||
make logs # Show recent logs from both servers
|
||||
make logs-primary # Follow primary logs in real-time
|
||||
make logs-remote # Follow remote logs in real-time
|
||||
```
|
||||
|
||||
### Check server health
|
||||
```bash
|
||||
make health
|
||||
```
|
||||
|
||||
### Clean up and retry
|
||||
```bash
|
||||
make clean
|
||||
make test-with-server
|
||||
```
|
||||
@@ -1,375 +0,0 @@
|
||||
package remote_cache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// Test configuration
|
||||
// Uses two SeaweedFS instances:
|
||||
// - Primary: The one being tested (has remote caching)
|
||||
// - Remote: Acts as the "remote" S3 storage
|
||||
const (
|
||||
// Primary SeaweedFS
|
||||
primaryEndpoint = "http://localhost:8333"
|
||||
primaryMasterPort = "9333"
|
||||
|
||||
// Remote SeaweedFS (acts as remote storage)
|
||||
remoteEndpoint = "http://localhost:8334"
|
||||
|
||||
// Credentials (anonymous access for testing)
|
||||
accessKey = "any"
|
||||
secretKey = "any"
|
||||
|
||||
// Bucket name - mounted on primary as remote storage
|
||||
testBucket = "remotemounted"
|
||||
|
||||
// Path to weed binary
|
||||
weedBinary = "../../../weed/weed_binary"
|
||||
)
|
||||
|
||||
var (
|
||||
primaryClient *s3.S3
|
||||
primaryClientOnce sync.Once
|
||||
)
|
||||
|
||||
func getPrimaryClient() *s3.S3 {
|
||||
primaryClientOnce.Do(func() {
|
||||
primaryClient = createS3Client(primaryEndpoint)
|
||||
})
|
||||
return primaryClient
|
||||
}
|
||||
|
||||
func createS3Client(endpoint string) *s3.S3 {
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Region: aws.String("us-east-1"),
|
||||
Endpoint: aws.String(endpoint),
|
||||
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
|
||||
DisableSSL: aws.Bool(!strings.HasPrefix(endpoint, "https")),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("failed to create session: %v", err))
|
||||
}
|
||||
return s3.New(sess)
|
||||
}
|
||||
|
||||
// skipIfNotRunning skips the test if the servers aren't running
|
||||
func skipIfNotRunning(t *testing.T) {
|
||||
resp, err := http.Get(primaryEndpoint)
|
||||
if err != nil {
|
||||
t.Skipf("Primary SeaweedFS not running at %s: %v", primaryEndpoint, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
resp, err = http.Get(remoteEndpoint)
|
||||
if err != nil {
|
||||
t.Skipf("Remote SeaweedFS not running at %s: %v", remoteEndpoint, err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
}
|
||||
|
||||
// runWeedShell executes a weed shell command
|
||||
func runWeedShell(t *testing.T, command string) (string, error) {
|
||||
cmd := exec.Command(weedBinary, "shell", "-master=localhost:"+primaryMasterPort)
|
||||
cmd.Stdin = strings.NewReader(command + "\nexit\n")
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Logf("weed shell command '%s' failed: %v, output: %s", command, err, string(output))
|
||||
return string(output), err
|
||||
}
|
||||
return string(output), nil
|
||||
}
|
||||
|
||||
// uploadToPrimary uploads an object to the primary SeaweedFS (local write)
|
||||
func uploadToPrimary(t *testing.T, key string, data []byte) {
|
||||
_, err := getPrimaryClient().PutObject(&s3.PutObjectInput{
|
||||
Bucket: aws.String(testBucket),
|
||||
Key: aws.String(key),
|
||||
Body: bytes.NewReader(data),
|
||||
})
|
||||
require.NoError(t, err, "failed to upload to primary SeaweedFS")
|
||||
}
|
||||
|
||||
// getFromPrimary gets an object from primary SeaweedFS
|
||||
func getFromPrimary(t *testing.T, key string) []byte {
|
||||
resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
|
||||
Bucket: aws.String(testBucket),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "failed to get from primary SeaweedFS")
|
||||
defer resp.Body.Close()
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err, "failed to read response body")
|
||||
return data
|
||||
}
|
||||
|
||||
// syncToRemote syncs local data to remote storage
|
||||
func syncToRemote(t *testing.T) {
|
||||
t.Log("Syncing to remote storage...")
|
||||
output, err := runWeedShell(t, "remote.cache.uncache -dir=/buckets/"+testBucket+" -include=*")
|
||||
if err != nil {
|
||||
t.Logf("syncToRemote warning: %v", err)
|
||||
}
|
||||
t.Log(output)
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
// uncacheLocal purges the local cache, forcing data to be fetched from remote
|
||||
func uncacheLocal(t *testing.T, pattern string) {
|
||||
t.Logf("Purging local cache for pattern: %s", pattern)
|
||||
output, err := runWeedShell(t, fmt.Sprintf("remote.uncache -dir=/buckets/%s -include=%s", testBucket, pattern))
|
||||
if err != nil {
|
||||
t.Logf("uncacheLocal warning: %v", err)
|
||||
}
|
||||
t.Log(output)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
// TestRemoteCacheBasic tests the basic caching workflow:
|
||||
// 1. Write to local
|
||||
// 2. Uncache (push to remote, remove local chunks)
|
||||
// 3. Read (triggers caching from remote)
|
||||
func TestRemoteCacheBasic(t *testing.T) {
|
||||
skipIfNotRunning(t)
|
||||
|
||||
testKey := fmt.Sprintf("test-basic-%d.txt", time.Now().UnixNano())
|
||||
testData := []byte("Hello, this is test data for remote caching!")
|
||||
|
||||
// Step 1: Write to local
|
||||
t.Log("Step 1: Writing object to primary SeaweedFS (local)...")
|
||||
uploadToPrimary(t, testKey, testData)
|
||||
|
||||
// Verify it's readable
|
||||
result := getFromPrimary(t, testKey)
|
||||
assert.Equal(t, testData, result, "initial read mismatch")
|
||||
|
||||
// Step 2: Uncache - push to remote and remove local chunks
|
||||
t.Log("Step 2: Uncaching (pushing to remote, removing local chunks)...")
|
||||
uncacheLocal(t, testKey)
|
||||
|
||||
// Step 3: Read - this should trigger caching from remote
|
||||
t.Log("Step 3: Reading object (should trigger caching from remote)...")
|
||||
start := time.Now()
|
||||
result = getFromPrimary(t, testKey)
|
||||
firstReadDuration := time.Since(start)
|
||||
|
||||
assert.Equal(t, testData, result, "data mismatch after cache")
|
||||
t.Logf("First read (from remote) took %v", firstReadDuration)
|
||||
|
||||
// Step 4: Read again - should be from local cache
|
||||
t.Log("Step 4: Reading again (should be from local cache)...")
|
||||
start = time.Now()
|
||||
result = getFromPrimary(t, testKey)
|
||||
secondReadDuration := time.Since(start)
|
||||
|
||||
assert.Equal(t, testData, result, "data mismatch on cached read")
|
||||
t.Logf("Second read (from cache) took %v", secondReadDuration)
|
||||
|
||||
t.Log("Basic caching test passed")
|
||||
}
|
||||
|
||||
// TestRemoteCacheConcurrent tests that concurrent reads of the same
|
||||
// remote object only trigger ONE caching operation (singleflight deduplication)
|
||||
func TestRemoteCacheConcurrent(t *testing.T) {
|
||||
skipIfNotRunning(t)
|
||||
|
||||
testKey := fmt.Sprintf("test-concurrent-%d.txt", time.Now().UnixNano())
|
||||
// Use larger data to make caching take measurable time
|
||||
testData := make([]byte, 1024*1024) // 1MB
|
||||
for i := range testData {
|
||||
testData[i] = byte(i % 256)
|
||||
}
|
||||
|
||||
// Step 1: Write to local
|
||||
t.Log("Step 1: Writing 1MB object to primary SeaweedFS...")
|
||||
uploadToPrimary(t, testKey, testData)
|
||||
|
||||
// Verify it's readable
|
||||
result := getFromPrimary(t, testKey)
|
||||
assert.Equal(t, len(testData), len(result), "initial size mismatch")
|
||||
|
||||
// Step 2: Uncache
|
||||
t.Log("Step 2: Uncaching (pushing to remote)...")
|
||||
uncacheLocal(t, testKey)
|
||||
|
||||
// Step 3: Launch many concurrent reads - singleflight should deduplicate
|
||||
numRequests := 10
|
||||
var wg sync.WaitGroup
|
||||
var successCount atomic.Int32
|
||||
var errorCount atomic.Int32
|
||||
results := make(chan []byte, numRequests)
|
||||
|
||||
t.Logf("Step 3: Launching %d concurrent requests...", numRequests)
|
||||
startTime := time.Now()
|
||||
|
||||
for i := 0; i < numRequests; i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
|
||||
Bucket: aws.String(testBucket),
|
||||
Key: aws.String(testKey),
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("Request %d failed: %v", idx, err)
|
||||
errorCount.Add(1)
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Logf("Request %d read failed: %v", idx, err)
|
||||
errorCount.Add(1)
|
||||
return
|
||||
}
|
||||
|
||||
results <- data
|
||||
successCount.Add(1)
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
close(results)
|
||||
totalDuration := time.Since(startTime)
|
||||
|
||||
t.Logf("All %d requests completed in %v", numRequests, totalDuration)
|
||||
t.Logf("Successful: %d, Failed: %d", successCount.Load(), errorCount.Load())
|
||||
|
||||
// Verify all successful requests returned correct data
|
||||
for data := range results {
|
||||
assert.Equal(t, len(testData), len(data), "data length mismatch")
|
||||
}
|
||||
|
||||
// All requests should succeed
|
||||
assert.Equal(t, int32(numRequests), successCount.Load(), "some requests failed")
|
||||
assert.Equal(t, int32(0), errorCount.Load(), "no requests should fail")
|
||||
|
||||
t.Log("Concurrent caching test passed")
|
||||
}
|
||||
|
||||
// TestRemoteCacheLargeObject tests caching of larger objects
|
||||
func TestRemoteCacheLargeObject(t *testing.T) {
|
||||
skipIfNotRunning(t)
|
||||
|
||||
testKey := fmt.Sprintf("test-large-%d.bin", time.Now().UnixNano())
|
||||
// 5MB object
|
||||
testData := make([]byte, 5*1024*1024)
|
||||
for i := range testData {
|
||||
testData[i] = byte(i % 256)
|
||||
}
|
||||
|
||||
// Step 1: Write to local
|
||||
t.Log("Step 1: Writing 5MB object to primary SeaweedFS...")
|
||||
uploadToPrimary(t, testKey, testData)
|
||||
|
||||
// Verify it's readable
|
||||
result := getFromPrimary(t, testKey)
|
||||
assert.Equal(t, len(testData), len(result), "initial size mismatch")
|
||||
|
||||
// Step 2: Uncache
|
||||
t.Log("Step 2: Uncaching...")
|
||||
uncacheLocal(t, testKey)
|
||||
|
||||
// Step 3: Read from remote
|
||||
t.Log("Step 3: Reading 5MB object (should cache from remote)...")
|
||||
start := time.Now()
|
||||
result = getFromPrimary(t, testKey)
|
||||
duration := time.Since(start)
|
||||
|
||||
assert.Equal(t, len(testData), len(result), "size mismatch")
|
||||
assert.Equal(t, testData, result, "data mismatch")
|
||||
t.Logf("Large object cached in %v", duration)
|
||||
|
||||
t.Log("Large object caching test passed")
|
||||
}
|
||||
|
||||
// TestRemoteCacheRangeRequest tests that range requests work after caching
|
||||
func TestRemoteCacheRangeRequest(t *testing.T) {
|
||||
skipIfNotRunning(t)
|
||||
|
||||
testKey := fmt.Sprintf("test-range-%d.txt", time.Now().UnixNano())
|
||||
testData := []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
||||
|
||||
// Write, uncache, then test range request
|
||||
t.Log("Writing and uncaching object...")
|
||||
uploadToPrimary(t, testKey, testData)
|
||||
uncacheLocal(t, testKey)
|
||||
|
||||
// Range request should work and trigger caching
|
||||
t.Log("Testing range request (bytes 10-19)...")
|
||||
resp, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
|
||||
Bucket: aws.String(testBucket),
|
||||
Key: aws.String(testKey),
|
||||
Range: aws.String("bytes=10-19"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
rangeData, err := io.ReadAll(resp.Body)
|
||||
require.NoError(t, err)
|
||||
|
||||
expected := testData[10:20] // "ABCDEFGHIJ"
|
||||
assert.Equal(t, expected, rangeData, "range data mismatch")
|
||||
t.Logf("Range request returned: %s", string(rangeData))
|
||||
|
||||
t.Log("Range request test passed")
|
||||
}
|
||||
|
||||
// TestRemoteCacheNotFound tests that non-existent objects return proper errors
|
||||
func TestRemoteCacheNotFound(t *testing.T) {
|
||||
skipIfNotRunning(t)
|
||||
|
||||
testKey := fmt.Sprintf("non-existent-object-%d", time.Now().UnixNano())
|
||||
|
||||
_, err := getPrimaryClient().GetObject(&s3.GetObjectInput{
|
||||
Bucket: aws.String(testBucket),
|
||||
Key: aws.String(testKey),
|
||||
})
|
||||
|
||||
assert.Error(t, err, "should get error for non-existent object")
|
||||
t.Logf("Got expected error: %v", err)
|
||||
|
||||
t.Log("Not found test passed")
|
||||
}
|
||||
|
||||
// TestMain sets up and tears down the test environment
|
||||
func TestMain(m *testing.M) {
|
||||
if !isServerRunning(primaryEndpoint) {
|
||||
fmt.Println("WARNING: Primary SeaweedFS not running at", primaryEndpoint)
|
||||
fmt.Println(" Run 'make test-with-server' to start servers automatically")
|
||||
}
|
||||
if !isServerRunning(remoteEndpoint) {
|
||||
fmt.Println("WARNING: Remote SeaweedFS not running at", remoteEndpoint)
|
||||
fmt.Println(" Run 'make test-with-server' to start servers automatically")
|
||||
}
|
||||
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func isServerRunning(url string) bool {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
resp.Body.Close()
|
||||
return true
|
||||
}
|
||||
@@ -1,505 +0,0 @@
|
||||
package sse_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestGitHub7562CopyFromEncryptedToTempToEncrypted reproduces the exact scenario from
|
||||
// GitHub issue #7562: copying from an encrypted bucket to a temp bucket, then to another
|
||||
// encrypted bucket fails with InternalError.
|
||||
//
|
||||
// Reproduction steps:
|
||||
// 1. Create source bucket with SSE-S3 encryption enabled
|
||||
// 2. Upload object (automatically encrypted)
|
||||
// 3. Create temp bucket (no encryption)
|
||||
// 4. Copy object from source to temp (decrypts)
|
||||
// 5. Delete source bucket
|
||||
// 6. Create destination bucket with SSE-S3 encryption
|
||||
// 7. Copy object from temp to dest (should re-encrypt) - THIS FAILS
|
||||
func TestGitHub7562CopyFromEncryptedToTempToEncrypted(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client, err := createS3Client(ctx, defaultConfig)
|
||||
require.NoError(t, err, "Failed to create S3 client")
|
||||
|
||||
// Create three buckets
|
||||
srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-src-")
|
||||
require.NoError(t, err, "Failed to create source bucket")
|
||||
|
||||
tempBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-temp-")
|
||||
require.NoError(t, err, "Failed to create temp bucket")
|
||||
|
||||
destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-dest-")
|
||||
require.NoError(t, err, "Failed to create destination bucket")
|
||||
|
||||
// Cleanup at the end
|
||||
defer func() {
|
||||
// Clean up in reverse order of creation
|
||||
cleanupTestBucket(ctx, client, destBucket)
|
||||
cleanupTestBucket(ctx, client, tempBucket)
|
||||
// Note: srcBucket is deleted during the test
|
||||
}()
|
||||
|
||||
testData := []byte("Test data for GitHub issue #7562 - copy from encrypted to temp to encrypted bucket")
|
||||
objectKey := "demo-file.txt"
|
||||
|
||||
t.Logf("[1] Creating source bucket with SSE-S3 default encryption: %s", srcBucket)
|
||||
|
||||
// Step 1: Enable SSE-S3 default encryption on source bucket
|
||||
_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
|
||||
Bucket: aws.String(srcBucket),
|
||||
ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
|
||||
Rules: []types.ServerSideEncryptionRule{
|
||||
{
|
||||
ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
|
||||
SSEAlgorithm: types.ServerSideEncryptionAes256,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to set source bucket default encryption")
|
||||
|
||||
t.Log("[2] Uploading demo object to source bucket")
|
||||
|
||||
// Step 2: Upload object to source bucket (will be automatically encrypted)
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(srcBucket),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(testData),
|
||||
// No encryption header - bucket default applies
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload to source bucket")
|
||||
|
||||
// Verify source object is encrypted
|
||||
srcHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(srcBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to HEAD source object")
|
||||
assert.Equal(t, types.ServerSideEncryptionAes256, srcHead.ServerSideEncryption,
|
||||
"Source object should be SSE-S3 encrypted")
|
||||
t.Logf("Source object encryption: %v", srcHead.ServerSideEncryption)
|
||||
|
||||
t.Logf("[3] Creating temp bucket (no encryption): %s", tempBucket)
|
||||
// Temp bucket already created without encryption
|
||||
|
||||
t.Log("[4] Copying object from source to temp (should decrypt)")
|
||||
|
||||
// Step 4: Copy to temp bucket (no encryption = decrypts)
|
||||
_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(tempBucket),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)),
|
||||
// No encryption header - data stored unencrypted
|
||||
})
|
||||
require.NoError(t, err, "Failed to copy to temp bucket")
|
||||
|
||||
// Verify temp object is NOT encrypted
|
||||
tempHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(tempBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to HEAD temp object")
|
||||
assert.Empty(t, tempHead.ServerSideEncryption, "Temp object should NOT be encrypted")
|
||||
t.Logf("Temp object encryption: %v (should be empty)", tempHead.ServerSideEncryption)
|
||||
|
||||
// Verify temp object content
|
||||
tempGet, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(tempBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to GET temp object")
|
||||
tempData, err := io.ReadAll(tempGet.Body)
|
||||
tempGet.Body.Close()
|
||||
require.NoError(t, err, "Failed to read temp object")
|
||||
assertDataEqual(t, testData, tempData, "Temp object data should match original")
|
||||
|
||||
t.Log("[5] Deleting original source bucket")
|
||||
|
||||
// Step 5: Delete source bucket
|
||||
err = cleanupTestBucket(ctx, client, srcBucket)
|
||||
require.NoError(t, err, "Failed to delete source bucket")
|
||||
|
||||
t.Logf("[6] Creating destination bucket with SSE-S3 encryption: %s", destBucket)
|
||||
|
||||
// Step 6: Enable SSE-S3 default encryption on destination bucket
|
||||
_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
|
||||
Rules: []types.ServerSideEncryptionRule{
|
||||
{
|
||||
ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
|
||||
SSEAlgorithm: types.ServerSideEncryptionAes256,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to set destination bucket default encryption")
|
||||
|
||||
t.Log("[7] Copying object from temp to dest (should re-encrypt) - THIS IS WHERE #7562 FAILS")
|
||||
|
||||
// Step 7: Copy from temp to dest bucket (should re-encrypt with SSE-S3)
|
||||
// THIS IS THE STEP THAT FAILS IN GITHUB ISSUE #7562
|
||||
_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(fmt.Sprintf("%s/%s", tempBucket, objectKey)),
|
||||
// No encryption header - bucket default should apply
|
||||
})
|
||||
require.NoError(t, err, "GitHub #7562: Failed to copy from temp to encrypted dest bucket")
|
||||
|
||||
// Verify destination object is encrypted
|
||||
destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to HEAD destination object")
|
||||
assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption,
|
||||
"Destination object should be SSE-S3 encrypted via bucket default")
|
||||
t.Logf("Destination object encryption: %v", destHead.ServerSideEncryption)
|
||||
|
||||
// Verify destination object content is correct
|
||||
destGet, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to GET destination object")
|
||||
destData, err := io.ReadAll(destGet.Body)
|
||||
destGet.Body.Close()
|
||||
require.NoError(t, err, "Failed to read destination object")
|
||||
assertDataEqual(t, testData, destData, "GitHub #7562: Destination object data mismatch after re-encryption")
|
||||
|
||||
t.Log("[done] GitHub #7562 reproduction test completed successfully!")
|
||||
}
|
||||
|
||||
// TestGitHub7562SimpleScenario tests the simpler variant: just copy unencrypted to encrypted bucket
|
||||
func TestGitHub7562SimpleScenario(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client, err := createS3Client(ctx, defaultConfig)
|
||||
require.NoError(t, err, "Failed to create S3 client")
|
||||
|
||||
// Create two buckets
|
||||
srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-simple-src-")
|
||||
require.NoError(t, err, "Failed to create source bucket")
|
||||
defer cleanupTestBucket(ctx, client, srcBucket)
|
||||
|
||||
destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-simple-dest-")
|
||||
require.NoError(t, err, "Failed to create destination bucket")
|
||||
defer cleanupTestBucket(ctx, client, destBucket)
|
||||
|
||||
testData := []byte("Simple test for unencrypted to encrypted copy")
|
||||
objectKey := "test-object.txt"
|
||||
|
||||
t.Logf("Source bucket (no encryption): %s", srcBucket)
|
||||
t.Logf("Dest bucket (SSE-S3 default): %s", destBucket)
|
||||
|
||||
// Upload to unencrypted source bucket
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(srcBucket),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(testData),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload to source bucket")
|
||||
|
||||
// Enable SSE-S3 on destination bucket
|
||||
_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
|
||||
Rules: []types.ServerSideEncryptionRule{
|
||||
{
|
||||
ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
|
||||
SSEAlgorithm: types.ServerSideEncryptionAes256,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to set dest bucket encryption")
|
||||
|
||||
// Copy to encrypted bucket (should use bucket default encryption)
|
||||
_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)),
|
||||
})
|
||||
require.NoError(t, err, "Failed to copy to encrypted bucket")
|
||||
|
||||
// Verify destination is encrypted
|
||||
destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to HEAD dest object")
|
||||
assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption,
|
||||
"Object should be encrypted via bucket default")
|
||||
|
||||
// Verify content
|
||||
destGet, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to GET dest object")
|
||||
destData, err := io.ReadAll(destGet.Body)
|
||||
destGet.Body.Close()
|
||||
require.NoError(t, err, "Failed to read dest object")
|
||||
assertDataEqual(t, testData, destData, "Data mismatch")
|
||||
}
|
||||
|
||||
// TestGitHub7562DebugMetadata helps debug what metadata is present on objects at each step
|
||||
func TestGitHub7562DebugMetadata(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client, err := createS3Client(ctx, defaultConfig)
|
||||
require.NoError(t, err, "Failed to create S3 client")
|
||||
|
||||
// Create three buckets
|
||||
srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-debug-src-")
|
||||
require.NoError(t, err, "Failed to create source bucket")
|
||||
|
||||
tempBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-debug-temp-")
|
||||
require.NoError(t, err, "Failed to create temp bucket")
|
||||
|
||||
destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-debug-dest-")
|
||||
require.NoError(t, err, "Failed to create destination bucket")
|
||||
|
||||
defer func() {
|
||||
cleanupTestBucket(ctx, client, destBucket)
|
||||
cleanupTestBucket(ctx, client, tempBucket)
|
||||
}()
|
||||
|
||||
testData := []byte("Debug metadata test for GitHub #7562")
|
||||
objectKey := "debug-file.txt"
|
||||
|
||||
// Enable SSE-S3 on source
|
||||
_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
|
||||
Bucket: aws.String(srcBucket),
|
||||
ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
|
||||
Rules: []types.ServerSideEncryptionRule{
|
||||
{
|
||||
ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
|
||||
SSEAlgorithm: types.ServerSideEncryptionAes256,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to set source bucket encryption")
|
||||
|
||||
// Upload
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(srcBucket),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(testData),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload")
|
||||
|
||||
// Log source object headers
|
||||
srcHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(srcBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to HEAD source")
|
||||
t.Logf("=== SOURCE OBJECT (encrypted) ===")
|
||||
t.Logf("ServerSideEncryption: %v", srcHead.ServerSideEncryption)
|
||||
t.Logf("Metadata: %v", srcHead.Metadata)
|
||||
t.Logf("ContentLength: %d", aws.ToInt64(srcHead.ContentLength))
|
||||
|
||||
// Copy to temp
|
||||
_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(tempBucket),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)),
|
||||
})
|
||||
require.NoError(t, err, "Failed to copy to temp")
|
||||
|
||||
// Log temp object headers
|
||||
tempHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(tempBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to HEAD temp")
|
||||
t.Logf("=== TEMP OBJECT (should be unencrypted) ===")
|
||||
t.Logf("ServerSideEncryption: %v (should be empty)", tempHead.ServerSideEncryption)
|
||||
t.Logf("Metadata: %v", tempHead.Metadata)
|
||||
t.Logf("ContentLength: %d", aws.ToInt64(tempHead.ContentLength))
|
||||
|
||||
// Verify temp is NOT encrypted
|
||||
if tempHead.ServerSideEncryption != "" {
|
||||
t.Logf("WARNING: Temp object unexpectedly has encryption: %v", tempHead.ServerSideEncryption)
|
||||
}
|
||||
|
||||
// Delete source bucket
|
||||
cleanupTestBucket(ctx, client, srcBucket)
|
||||
|
||||
// Enable SSE-S3 on dest
|
||||
_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
|
||||
Rules: []types.ServerSideEncryptionRule{
|
||||
{
|
||||
ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
|
||||
SSEAlgorithm: types.ServerSideEncryptionAes256,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to set dest bucket encryption")
|
||||
|
||||
// Copy to dest - THIS IS WHERE #7562 FAILS
|
||||
t.Log("=== COPYING TO ENCRYPTED DEST ===")
|
||||
_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(fmt.Sprintf("%s/%s", tempBucket, objectKey)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Logf("!!! COPY FAILED (GitHub #7562): %v", err)
|
||||
t.FailNow()
|
||||
}
|
||||
|
||||
// Log dest object headers
|
||||
destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to HEAD dest")
|
||||
t.Logf("=== DEST OBJECT (should be encrypted) ===")
|
||||
t.Logf("ServerSideEncryption: %v", destHead.ServerSideEncryption)
|
||||
t.Logf("Metadata: %v", destHead.Metadata)
|
||||
t.Logf("ContentLength: %d", aws.ToInt64(destHead.ContentLength))
|
||||
|
||||
// Verify dest IS encrypted
|
||||
assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption,
|
||||
"Dest object should be encrypted")
|
||||
|
||||
// Verify content is readable
|
||||
destGet, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to GET dest")
|
||||
destData, err := io.ReadAll(destGet.Body)
|
||||
destGet.Body.Close()
|
||||
require.NoError(t, err, "Failed to read dest")
|
||||
assertDataEqual(t, testData, destData, "Data mismatch")
|
||||
|
||||
t.Log("=== DEBUG TEST PASSED ===")
|
||||
}
|
||||
|
||||
// TestGitHub7562LargeFile tests the issue with larger files that might trigger multipart handling
|
||||
func TestGitHub7562LargeFile(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client, err := createS3Client(ctx, defaultConfig)
|
||||
require.NoError(t, err, "Failed to create S3 client")
|
||||
|
||||
srcBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-large-src-")
|
||||
require.NoError(t, err, "Failed to create source bucket")
|
||||
|
||||
tempBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-large-temp-")
|
||||
require.NoError(t, err, "Failed to create temp bucket")
|
||||
|
||||
destBucket, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"7562-large-dest-")
|
||||
require.NoError(t, err, "Failed to create destination bucket")
|
||||
|
||||
defer func() {
|
||||
cleanupTestBucket(ctx, client, destBucket)
|
||||
cleanupTestBucket(ctx, client, tempBucket)
|
||||
}()
|
||||
|
||||
// Use larger file to potentially trigger different code paths
|
||||
testData := generateTestData(5 * 1024 * 1024) // 5MB
|
||||
objectKey := "large-file.bin"
|
||||
|
||||
t.Logf("Testing with %d byte file", len(testData))
|
||||
|
||||
// Enable SSE-S3 on source
|
||||
_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
|
||||
Bucket: aws.String(srcBucket),
|
||||
ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
|
||||
Rules: []types.ServerSideEncryptionRule{
|
||||
{
|
||||
ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
|
||||
SSEAlgorithm: types.ServerSideEncryptionAes256,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to set source bucket encryption")
|
||||
|
||||
// Upload
|
||||
_, err = client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: aws.String(srcBucket),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(testData),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload")
|
||||
|
||||
// Copy to temp (decrypt)
|
||||
_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(tempBucket),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(fmt.Sprintf("%s/%s", srcBucket, objectKey)),
|
||||
})
|
||||
require.NoError(t, err, "Failed to copy to temp")
|
||||
|
||||
// Delete source
|
||||
cleanupTestBucket(ctx, client, srcBucket)
|
||||
|
||||
// Enable SSE-S3 on dest
|
||||
_, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
|
||||
Rules: []types.ServerSideEncryptionRule{
|
||||
{
|
||||
ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
|
||||
SSEAlgorithm: types.ServerSideEncryptionAes256,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to set dest bucket encryption")
|
||||
|
||||
// Copy to dest (re-encrypt) - GitHub #7562
|
||||
_, err = client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
CopySource: aws.String(fmt.Sprintf("%s/%s", tempBucket, objectKey)),
|
||||
})
|
||||
require.NoError(t, err, "GitHub #7562: Large file copy to encrypted bucket failed")
|
||||
|
||||
// Verify
|
||||
destHead, err := client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to HEAD dest")
|
||||
assert.Equal(t, types.ServerSideEncryptionAes256, destHead.ServerSideEncryption)
|
||||
assert.Equal(t, int64(len(testData)), aws.ToInt64(destHead.ContentLength))
|
||||
|
||||
// Verify content
|
||||
destGet, err := client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: aws.String(destBucket),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to GET dest")
|
||||
destData, err := io.ReadAll(destGet.Body)
|
||||
destGet.Body.Close()
|
||||
require.NoError(t, err, "Failed to read dest")
|
||||
assertDataEqual(t, testData, destData, "Large file data mismatch")
|
||||
|
||||
t.Log("Large file test passed!")
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ start-server: check-deps
|
||||
@echo "🔍 DEBUG: Creating volume directory..."
|
||||
@mkdir -p ./test-volume-data
|
||||
@echo "🔍 DEBUG: Launching SeaweedFS server in background..."
|
||||
@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -master.peers=none -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
|
||||
@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
|
||||
@$(WEED_BINARY) server \
|
||||
-debug \
|
||||
-s3 \
|
||||
@@ -91,7 +91,6 @@ start-server: check-deps
|
||||
-filer \
|
||||
-filer.maxMB=64 \
|
||||
-master.volumeSizeLimitMB=50 \
|
||||
-master.peers=none \
|
||||
-volume.max=100 \
|
||||
-dir=./test-volume-data \
|
||||
-volume.preStopSeconds=1 \
|
||||
@@ -222,13 +221,13 @@ test-with-server: start-server
|
||||
test-versioning-with-configs: check-deps
|
||||
@echo "Testing with different S3 configurations..."
|
||||
@echo "Testing with empty folder allowed..."
|
||||
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
|
||||
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
|
||||
@sleep 5
|
||||
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
|
||||
@if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi
|
||||
@sleep 2
|
||||
@echo "Testing with delete bucket not empty disabled..."
|
||||
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
|
||||
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
|
||||
@sleep 5
|
||||
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
|
||||
@if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi
|
||||
@@ -240,13 +239,6 @@ test-versioning-stress: check-deps
|
||||
@go test -v -timeout=20m -run "TestVersioningConcurrentOperations" . -count=5
|
||||
@echo "✅ Stress tests completed"
|
||||
|
||||
# Pagination stress testing (tests >1000 versions)
|
||||
test-versioning-pagination-stress: check-deps
|
||||
@echo "Running pagination stress tests (>1000 versions)..."
|
||||
@echo "This test creates 1500+ versions and may take several minutes..."
|
||||
@ENABLE_STRESS_TESTS=true go test -v -timeout=30m -run "TestVersioningPagination" .
|
||||
@echo "✅ Pagination stress tests completed"
|
||||
|
||||
# Generate test reports
|
||||
test-report: check-deps
|
||||
@echo "Generating test reports..."
|
||||
@@ -280,7 +272,6 @@ debug-server:
|
||||
-filer \
|
||||
-filer.maxMB=16 \
|
||||
-master.volumeSizeLimitMB=50 \
|
||||
-master.peers=none \
|
||||
-volume.max=100 \
|
||||
-dir=./test-volume-data \
|
||||
-volume.preStopSeconds=1 \
|
||||
@@ -329,7 +320,6 @@ start-server-simple: check-deps
|
||||
-filer \
|
||||
-filer.maxMB=64 \
|
||||
-master.volumeSizeLimitMB=50 \
|
||||
-master.peers=none \
|
||||
-volume.max=100 \
|
||||
-volume.preStopSeconds=1 \
|
||||
-metricsPort=9324 \
|
||||
|
||||
@@ -1,327 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestVeleroKopiaVersionedObjectListing tests the exact access patterns used by Velero/Kopia
|
||||
// when backing up to a versioned S3 bucket. This test validates the fix for:
|
||||
// https://github.com/seaweedfs/seaweedfs/discussions/7573
|
||||
//
|
||||
// The bug was that when listing versioned objects with nested paths like
|
||||
// "kopia/logpaste/kopia.blobcfg", the returned Key would be doubled:
|
||||
// "kopia/logpaste/kopia/logpaste/kopia.blobcfg" instead of the correct path.
|
||||
//
|
||||
// This was caused by getLatestVersionEntryForListOperation setting the entry's
|
||||
// Name to the full path instead of just the base filename.
|
||||
func TestVeleroKopiaVersionedObjectListing(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Step 1: Create bucket
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// Step 2: Enable versioning (required to reproduce the bug)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Step 3: Create objects matching Velero/Kopia's typical path structure
|
||||
// These are the actual paths Kopia uses when storing repository data
|
||||
testObjects := map[string]string{
|
||||
"kopia/logpaste/kopia.blobcfg": "blob-config-content",
|
||||
"kopia/logpaste/kopia.repository": "repository-content",
|
||||
"kopia/logpaste/.storageconfig": "storage-config-content",
|
||||
"backups/namespace1/backup.tar": "backup-content",
|
||||
"restic/locks/lock-001.json": "lock-content",
|
||||
"restic/index/index-001.json": "index-content",
|
||||
"data/a/b/c/deeply-nested.dat": "nested-content",
|
||||
"single-file.txt": "single-file-content",
|
||||
}
|
||||
|
||||
// Upload all test objects
|
||||
for key, content := range testObjects {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
Body: strings.NewReader(content),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload object %s", key)
|
||||
}
|
||||
|
||||
// Step 4: Test listing with various prefixes (mimicking Kopia's ListObjects calls)
|
||||
testCases := []struct {
|
||||
name string
|
||||
prefix string
|
||||
expectedKeys []string
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "kopia repository prefix",
|
||||
prefix: "kopia/logpaste/",
|
||||
expectedKeys: []string{
|
||||
"kopia/logpaste/.storageconfig",
|
||||
"kopia/logpaste/kopia.blobcfg",
|
||||
"kopia/logpaste/kopia.repository",
|
||||
},
|
||||
description: "Kopia lists its repository directory to find config files",
|
||||
},
|
||||
{
|
||||
name: "exact config file",
|
||||
prefix: "kopia/logpaste/kopia.blobcfg",
|
||||
expectedKeys: []string{"kopia/logpaste/kopia.blobcfg"},
|
||||
description: "Kopia checks for specific config files",
|
||||
},
|
||||
{
|
||||
name: "restic index directory",
|
||||
prefix: "restic/index/",
|
||||
expectedKeys: []string{
|
||||
"restic/index/index-001.json",
|
||||
},
|
||||
description: "Restic lists index files (similar pattern to Kopia)",
|
||||
},
|
||||
{
|
||||
name: "deeply nested path",
|
||||
prefix: "data/a/b/c/",
|
||||
expectedKeys: []string{
|
||||
"data/a/b/c/deeply-nested.dat",
|
||||
},
|
||||
description: "Tests deeply nested paths don't get doubled",
|
||||
},
|
||||
{
|
||||
name: "backup directory",
|
||||
prefix: "backups/",
|
||||
expectedKeys: []string{
|
||||
"backups/namespace1/backup.tar",
|
||||
},
|
||||
description: "Velero lists backup directories",
|
||||
},
|
||||
{
|
||||
name: "root level single file",
|
||||
prefix: "single-file",
|
||||
expectedKeys: []string{
|
||||
"single-file.txt",
|
||||
},
|
||||
description: "Files at bucket root should work correctly",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// List objects with prefix
|
||||
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(tc.prefix),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Extract returned keys
|
||||
var actualKeys []string
|
||||
for _, obj := range listResp.Contents {
|
||||
key := *obj.Key
|
||||
actualKeys = append(actualKeys, key)
|
||||
|
||||
// Critical assertion: verify no path doubling
|
||||
// The bug would cause "kopia/logpaste/kopia.blobcfg" to become
|
||||
// "kopia/logpaste/kopia/logpaste/kopia.blobcfg"
|
||||
assert.False(t, hasDoubledPath(key),
|
||||
"Path doubling detected! Key '%s' should not have repeated path segments", key)
|
||||
|
||||
// Also verify we can actually GET the object using the listed key
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Should be able to GET object using listed key: %s", key)
|
||||
getResp.Body.Close()
|
||||
}
|
||||
|
||||
// Verify we got the expected keys
|
||||
assert.ElementsMatch(t, tc.expectedKeys, actualKeys,
|
||||
"%s: Listed keys should match expected", tc.description)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestVeleroKopiaGetAfterList simulates Kopia's pattern of listing and then getting objects
|
||||
// This specifically tests the bug where listed paths couldn't be retrieved
|
||||
func TestVeleroKopiaGetAfterList(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
// Create a nested object like Kopia does
|
||||
objectKey := "kopia/appwrite/kopia.repository"
|
||||
objectContent := "kopia-repository-config-data"
|
||||
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: strings.NewReader(objectContent),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// List objects (this is where the bug manifested - returned wrong Key)
|
||||
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String("kopia/appwrite/"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, listResp.Contents, 1, "Should list exactly one object")
|
||||
|
||||
listedKey := *listResp.Contents[0].Key
|
||||
|
||||
// The bug caused listedKey to be "kopia/appwrite/kopia/appwrite/kopia.repository"
|
||||
// instead of "kopia/appwrite/kopia.repository"
|
||||
assert.Equal(t, objectKey, listedKey,
|
||||
"Listed key should match the original key without path doubling")
|
||||
|
||||
// Now try to GET the object using the listed key
|
||||
// With the bug, this would fail because the doubled path doesn't exist
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(listedKey),
|
||||
})
|
||||
require.NoError(t, err, "Should be able to GET object using the key returned from ListObjects")
|
||||
defer getResp.Body.Close()
|
||||
|
||||
// Verify content matches
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, objectContent, string(body),
|
||||
"Retrieved content should match original")
|
||||
}
|
||||
|
||||
// TestVeleroMultipleVersionsWithNestedPaths tests listing when objects have multiple versions
|
||||
func TestVeleroMultipleVersionsWithNestedPaths(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
objectKey := "kopia/logpaste/kopia.blobcfg"
|
||||
|
||||
// Create multiple versions of the same object (Kopia updates config files)
|
||||
versions := []string{"version-1", "version-2", "version-3"}
|
||||
for _, content := range versions {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: strings.NewReader(content),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// List objects - should return only the latest version in regular listing
|
||||
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String("kopia/logpaste/"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should only see one object (the latest version)
|
||||
require.Len(t, listResp.Contents, 1, "Should list only one object (latest version)")
|
||||
|
||||
listedKey := *listResp.Contents[0].Key
|
||||
assert.Equal(t, objectKey, listedKey,
|
||||
"Listed key should match original without path doubling")
|
||||
assert.False(t, hasDoubledPath(listedKey),
|
||||
"Path should not be doubled")
|
||||
|
||||
// Verify we can GET the latest version
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(listedKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
defer getResp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "version-3", string(body),
|
||||
"Should get the latest version content")
|
||||
}
|
||||
|
||||
// TestVeleroListVersionsWithNestedPaths tests ListObjectVersions with nested paths
|
||||
func TestVeleroListVersionsWithNestedPaths(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
objectKey := "backups/velero/test-backup/backup.json"
|
||||
|
||||
// Create multiple versions
|
||||
for i := 1; i <= 3; i++ {
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: strings.NewReader(strings.Repeat("x", i*100)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// List all versions
|
||||
listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String("backups/velero/"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should have 3 versions
|
||||
require.Len(t, listResp.Versions, 3, "Should have 3 versions")
|
||||
|
||||
// Verify all version keys are correct (not doubled)
|
||||
for _, version := range listResp.Versions {
|
||||
key := *version.Key
|
||||
assert.Equal(t, objectKey, key,
|
||||
"Version key should match original")
|
||||
assert.False(t, hasDoubledPath(key),
|
||||
"Version key should not have doubled path: %s", key)
|
||||
|
||||
// Verify we can GET each version by its version ID
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
VersionId: version.VersionId,
|
||||
})
|
||||
require.NoError(t, err, "Should be able to GET version %s", *version.VersionId)
|
||||
func() {
|
||||
defer getResp.Body.Close()
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// hasDoubledPath checks if a path has a repeated pair of segments, which indicates
|
||||
// the path doubling bug. For example: "kopia/logpaste/kopia/logpaste/file.txt" would return true.
|
||||
func hasDoubledPath(path string) bool {
|
||||
parts := strings.Split(path, "/")
|
||||
if len(parts) < 4 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for repeated pairs of segments, e.g., a/b/.../a/b.
|
||||
for i := 0; i < len(parts)-3; i++ {
|
||||
for j := i + 2; j < len(parts)-1; j++ {
|
||||
if parts[i] == parts[j] && parts[i+1] == parts[j+1] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -1,521 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestMultipartUploadVersioningListETag tests that multipart uploaded objects
|
||||
// in versioned buckets have correct ETags when listed.
|
||||
// This covers a bug where synthetic entries for versioned objects didn't include
|
||||
// proper ETag handling for multipart uploads (ETags with format "<md5>-<parts>").
|
||||
func TestMultipartUploadVersioningListETag(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// Enable versioning
|
||||
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusEnabled,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to enable versioning")
|
||||
|
||||
// Create multipart upload
|
||||
objectKey := "multipart-test-object"
|
||||
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to create multipart upload")
|
||||
|
||||
uploadId := *createResp.UploadId
|
||||
|
||||
// Upload 2 parts (minimum 5MB per part except last)
|
||||
partSize := 5 * 1024 * 1024 // 5MB
|
||||
part1Data := bytes.Repeat([]byte("a"), partSize)
|
||||
part2Data := bytes.Repeat([]byte("b"), partSize)
|
||||
|
||||
// Calculate MD5 for each part
|
||||
part1MD5 := md5.Sum(part1Data)
|
||||
part2MD5 := md5.Sum(part2Data)
|
||||
|
||||
// Upload part 1
|
||||
uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadId),
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader(part1Data),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload part 1")
|
||||
|
||||
// Upload part 2
|
||||
uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadId),
|
||||
PartNumber: aws.Int32(2),
|
||||
Body: bytes.NewReader(part2Data),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload part 2")
|
||||
|
||||
// Complete multipart upload
|
||||
completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadId),
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: []types.CompletedPart{
|
||||
{
|
||||
ETag: uploadPart1Resp.ETag,
|
||||
PartNumber: aws.Int32(1),
|
||||
},
|
||||
{
|
||||
ETag: uploadPart2Resp.ETag,
|
||||
PartNumber: aws.Int32(2),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to complete multipart upload")
|
||||
|
||||
// Verify the ETag from CompleteMultipartUpload has the multipart format (md5-parts)
|
||||
completeETag := strings.Trim(*completeResp.ETag, "\"")
|
||||
assert.Contains(t, completeETag, "-", "Multipart ETag should contain '-' (format: md5-parts)")
|
||||
assert.True(t, strings.HasSuffix(completeETag, "-2"), "Multipart ETag should end with '-2' for 2 parts")
|
||||
|
||||
t.Logf("CompleteMultipartUpload ETag: %s", completeETag)
|
||||
t.Logf("Part 1 MD5: %x", part1MD5)
|
||||
t.Logf("Part 2 MD5: %x", part2MD5)
|
||||
|
||||
// HeadObject should return the same ETag
|
||||
headResp, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to head object")
|
||||
|
||||
headETag := strings.Trim(*headResp.ETag, "\"")
|
||||
assert.Equal(t, completeETag, headETag, "HeadObject ETag should match CompleteMultipartUpload ETag")
|
||||
|
||||
// ListObjectsV2 should return the same ETag
|
||||
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list objects")
|
||||
require.Len(t, listResp.Contents, 1, "Should have exactly one object")
|
||||
|
||||
listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
|
||||
assert.Equal(t, completeETag, listETag, "ListObjectsV2 ETag should match CompleteMultipartUpload ETag")
|
||||
assert.NotEmpty(t, listETag, "ListObjectsV2 ETag should not be empty")
|
||||
|
||||
t.Logf("ListObjectsV2 ETag: %s", listETag)
|
||||
|
||||
// ListObjectVersions should also return the correct ETag
|
||||
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list object versions")
|
||||
require.Len(t, versionsResp.Versions, 1, "Should have exactly one version")
|
||||
|
||||
versionETag := strings.Trim(*versionsResp.Versions[0].ETag, "\"")
|
||||
assert.Equal(t, completeETag, versionETag, "ListObjectVersions ETag should match CompleteMultipartUpload ETag")
|
||||
assert.NotEmpty(t, versionETag, "ListObjectVersions ETag should not be empty")
|
||||
|
||||
t.Logf("ListObjectVersions ETag: %s", versionETag)
|
||||
}
|
||||
|
||||
// TestMultipartUploadMultipleVersionsListETag tests that multiple versions
|
||||
// of multipart uploaded objects all have correct ETags when listed.
|
||||
func TestMultipartUploadMultipleVersionsListETag(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// Enable versioning
|
||||
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusEnabled,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to enable versioning")
|
||||
|
||||
objectKey := "multipart-multi-version-object"
|
||||
partSize := 5 * 1024 * 1024 // 5MB
|
||||
var expectedETags []string
|
||||
|
||||
// Create 3 versions using multipart upload
|
||||
for version := 1; version <= 3; version++ {
|
||||
// Create multipart upload
|
||||
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to create multipart upload for version %d", version)
|
||||
|
||||
uploadId := *createResp.UploadId
|
||||
|
||||
// Create unique data for each version
|
||||
partData := bytes.Repeat([]byte(fmt.Sprintf("%d", version)), partSize)
|
||||
|
||||
// Upload single part (still results in multipart ETag format)
|
||||
uploadPartResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadId),
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader(partData),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload part for version %d", version)
|
||||
|
||||
// Complete multipart upload
|
||||
completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: aws.String(uploadId),
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: []types.CompletedPart{
|
||||
{
|
||||
ETag: uploadPartResp.ETag,
|
||||
PartNumber: aws.Int32(1),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to complete multipart upload for version %d", version)
|
||||
|
||||
etag := strings.Trim(*completeResp.ETag, "\"")
|
||||
expectedETags = append(expectedETags, etag)
|
||||
t.Logf("Version %d ETag: %s", version, etag)
|
||||
}
|
||||
|
||||
// ListObjectVersions should return all versions with correct ETags
|
||||
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list object versions")
|
||||
require.Len(t, versionsResp.Versions, 3, "Should have exactly 3 versions")
|
||||
|
||||
// Collect ETags from the listing
|
||||
var listedETags []string
|
||||
for _, v := range versionsResp.Versions {
|
||||
etag := strings.Trim(*v.ETag, "\"")
|
||||
listedETags = append(listedETags, etag)
|
||||
assert.NotEmpty(t, etag, "Version ETag should not be empty")
|
||||
assert.Contains(t, etag, "-", "Multipart ETag should contain '-'")
|
||||
}
|
||||
|
||||
t.Logf("Expected ETags: %v", expectedETags)
|
||||
t.Logf("Listed ETags: %v", listedETags)
|
||||
|
||||
// Verify all expected ETags are present (order may differ due to version ordering)
|
||||
assert.ElementsMatch(t, expectedETags, listedETags, "Listed ETags should match all expected ETags")
|
||||
|
||||
// Regular ListObjectsV2 should return only the latest version with correct ETag
|
||||
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list objects")
|
||||
require.Len(t, listResp.Contents, 1, "Should have exactly one object in regular listing")
|
||||
|
||||
listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
|
||||
// The latest version (version 3) should be the one shown
|
||||
assert.Equal(t, expectedETags[2], listETag, "ListObjectsV2 should show latest version's ETag")
|
||||
}
|
||||
|
||||
// TestMixedSingleAndMultipartVersionsListETag tests that a mix of
|
||||
// single-part and multipart uploaded versions all have correct ETags.
|
||||
func TestMixedSingleAndMultipartVersionsListETag(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// Enable versioning
|
||||
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusEnabled,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to enable versioning")
|
||||
|
||||
objectKey := "mixed-upload-versions"
|
||||
|
||||
// Version 1: Regular PutObject (single-part, pure MD5 ETag)
|
||||
content1 := []byte("This is version 1 content - single part upload")
|
||||
putResp1, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(content1),
|
||||
})
|
||||
require.NoError(t, err, "Failed to put version 1")
|
||||
etag1 := strings.Trim(*putResp1.ETag, "\"")
|
||||
assert.NotContains(t, etag1, "-", "Single-part ETag should not contain '-'")
|
||||
t.Logf("Version 1 (PutObject) ETag: %s", etag1)
|
||||
|
||||
// Version 2: Multipart upload
|
||||
partSize := 5 * 1024 * 1024
|
||||
partData := bytes.Repeat([]byte("x"), partSize)
|
||||
|
||||
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to create multipart upload")
|
||||
|
||||
uploadPartResp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: createResp.UploadId,
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader(partData),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload part")
|
||||
|
||||
completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: createResp.UploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: []types.CompletedPart{
|
||||
{
|
||||
ETag: uploadPartResp.ETag,
|
||||
PartNumber: aws.Int32(1),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to complete multipart upload")
|
||||
etag2 := strings.Trim(*completeResp.ETag, "\"")
|
||||
assert.Contains(t, etag2, "-", "Multipart ETag should contain '-'")
|
||||
t.Logf("Version 2 (Multipart) ETag: %s", etag2)
|
||||
|
||||
// Version 3: Another regular PutObject
|
||||
content3 := []byte("This is version 3 content - another single part upload")
|
||||
putResp3, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: bytes.NewReader(content3),
|
||||
})
|
||||
require.NoError(t, err, "Failed to put version 3")
|
||||
etag3 := strings.Trim(*putResp3.ETag, "\"")
|
||||
assert.NotContains(t, etag3, "-", "Single-part ETag should not contain '-'")
|
||||
t.Logf("Version 3 (PutObject) ETag: %s", etag3)
|
||||
|
||||
// ListObjectVersions should return all 3 versions with correct ETags
|
||||
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list object versions")
|
||||
require.Len(t, versionsResp.Versions, 3, "Should have exactly 3 versions")
|
||||
|
||||
var listedETags []string
|
||||
for _, v := range versionsResp.Versions {
|
||||
etag := strings.Trim(*v.ETag, "\"")
|
||||
assert.NotEmpty(t, etag, "Version ETag should not be empty")
|
||||
listedETags = append(listedETags, etag)
|
||||
t.Logf("Listed version %s ETag: %s, IsLatest: %v", *v.VersionId, etag, *v.IsLatest)
|
||||
}
|
||||
|
||||
// Verify all ETags were found
|
||||
assert.ElementsMatch(t, []string{etag1, etag2, etag3}, listedETags, "Listed ETags should match all expected ETags")
|
||||
|
||||
// Regular ListObjectsV2 should return only the latest (version 3)
|
||||
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list objects")
|
||||
require.Len(t, listResp.Contents, 1, "Should have exactly one object")
|
||||
|
||||
listETag := strings.Trim(*listResp.Contents[0].ETag, "\"")
|
||||
assert.Equal(t, etag3, listETag, "ListObjectsV2 should show latest version's ETag (version 3)")
|
||||
}
|
||||
|
||||
// TestMultipartUploadDeleteMarkerListBehavior tests that delete markers work correctly
|
||||
// with multipart uploaded objects in versioned buckets.
|
||||
func TestMultipartUploadDeleteMarkerListBehavior(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// Enable versioning
|
||||
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
VersioningConfiguration: &types.VersioningConfiguration{
|
||||
Status: types.BucketVersioningStatusEnabled,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to enable versioning")
|
||||
|
||||
objectKey := "multipart-delete-marker-test"
|
||||
partSize := 5 * 1024 * 1024 // 5MB
|
||||
|
||||
// Create multipart upload
|
||||
createResp, err := client.CreateMultipartUpload(context.TODO(), &s3.CreateMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to create multipart upload")
|
||||
|
||||
// Upload 2 parts
|
||||
part1Data := bytes.Repeat([]byte("a"), partSize)
|
||||
part2Data := bytes.Repeat([]byte("b"), partSize)
|
||||
|
||||
uploadPart1Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: createResp.UploadId,
|
||||
PartNumber: aws.Int32(1),
|
||||
Body: bytes.NewReader(part1Data),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload part 1")
|
||||
|
||||
uploadPart2Resp, err := client.UploadPart(context.TODO(), &s3.UploadPartInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: createResp.UploadId,
|
||||
PartNumber: aws.Int32(2),
|
||||
Body: bytes.NewReader(part2Data),
|
||||
})
|
||||
require.NoError(t, err, "Failed to upload part 2")
|
||||
|
||||
// Complete multipart upload
|
||||
completeResp, err := client.CompleteMultipartUpload(context.TODO(), &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
UploadId: createResp.UploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: []types.CompletedPart{
|
||||
{ETag: uploadPart1Resp.ETag, PartNumber: aws.Int32(1)},
|
||||
{ETag: uploadPart2Resp.ETag, PartNumber: aws.Int32(2)},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err, "Failed to complete multipart upload")
|
||||
|
||||
multipartETag := strings.Trim(*completeResp.ETag, "\"")
|
||||
multipartVersionId := *completeResp.VersionId
|
||||
t.Logf("Multipart upload completed: ETag=%s, VersionId=%s", multipartETag, multipartVersionId)
|
||||
|
||||
// Verify object is visible in ListObjectsV2
|
||||
listBeforeDelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list objects before delete")
|
||||
require.Len(t, listBeforeDelete.Contents, 1, "Object should be visible before delete")
|
||||
assert.Equal(t, multipartETag, strings.Trim(*listBeforeDelete.Contents[0].ETag, "\""),
|
||||
"Listed ETag should match multipart ETag before delete")
|
||||
|
||||
// Delete object (creates delete marker)
|
||||
deleteResp, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to delete object")
|
||||
require.NotNil(t, deleteResp.DeleteMarker, "Should create delete marker")
|
||||
assert.True(t, *deleteResp.DeleteMarker, "DeleteMarker should be true")
|
||||
require.NotNil(t, deleteResp.VersionId, "Delete marker should have version ID")
|
||||
|
||||
deleteMarkerVersionId := *deleteResp.VersionId
|
||||
t.Logf("Delete marker created: VersionId=%s", deleteMarkerVersionId)
|
||||
|
||||
// ListObjectsV2 should NOT show the object anymore
|
||||
listAfterDelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list objects after delete")
|
||||
assert.Empty(t, listAfterDelete.Contents, "Object should NOT be visible after delete marker")
|
||||
|
||||
// ListObjectVersions should show both the original version AND the delete marker
|
||||
versionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list object versions")
|
||||
|
||||
// Should have 1 version (the multipart object)
|
||||
require.Len(t, versionsResp.Versions, 1, "Should have exactly 1 version (the multipart object)")
|
||||
version := versionsResp.Versions[0]
|
||||
assert.Equal(t, multipartVersionId, *version.VersionId, "Version ID should match")
|
||||
assert.Equal(t, multipartETag, strings.Trim(*version.ETag, "\""), "Version ETag should match multipart ETag")
|
||||
assert.False(t, *version.IsLatest, "Multipart version should NOT be latest (delete marker is latest)")
|
||||
|
||||
// Should have 1 delete marker
|
||||
require.Len(t, versionsResp.DeleteMarkers, 1, "Should have exactly 1 delete marker")
|
||||
deleteMarker := versionsResp.DeleteMarkers[0]
|
||||
assert.Equal(t, deleteMarkerVersionId, *deleteMarker.VersionId, "Delete marker version ID should match")
|
||||
assert.True(t, *deleteMarker.IsLatest, "Delete marker should be latest")
|
||||
|
||||
t.Logf("ListObjectVersions: 1 version (ETag=%s), 1 delete marker", multipartETag)
|
||||
|
||||
// Access the specific version by version ID - should still work
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String(multipartVersionId),
|
||||
})
|
||||
require.NoError(t, err, "Should be able to get object by version ID after delete marker")
|
||||
defer getResp.Body.Close()
|
||||
|
||||
assert.Equal(t, multipartETag, strings.Trim(*getResp.ETag, "\""),
|
||||
"GetObject with version ID should return correct ETag")
|
||||
assert.Equal(t, int64(partSize*2), *getResp.ContentLength,
|
||||
"GetObject with version ID should return correct size")
|
||||
|
||||
t.Logf("Successfully retrieved version %s after delete marker", multipartVersionId)
|
||||
|
||||
// Delete the delete marker to "undelete" the object
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String(deleteMarkerVersionId),
|
||||
})
|
||||
require.NoError(t, err, "Failed to delete the delete marker")
|
||||
|
||||
// ListObjectsV2 should show the object again
|
||||
listAfterUndelete, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err, "Failed to list objects after undelete")
|
||||
require.Len(t, listAfterUndelete.Contents, 1, "Object should be visible again after removing delete marker")
|
||||
assert.Equal(t, multipartETag, strings.Trim(*listAfterUndelete.Contents[0].ETag, "\""),
|
||||
"Undeleted object should have correct multipart ETag")
|
||||
|
||||
t.Logf("Object restored after delete marker removal, ETag=%s", multipartETag)
|
||||
}
|
||||
|
||||
@@ -1,322 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestVersioningPaginationOver1000Versions tests that ListObjectVersions correctly
|
||||
// handles objects with more than 1000 versions using pagination.
|
||||
// This is a stress test that creates 1500 versions and verifies all are listed.
|
||||
//
|
||||
// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationOver1000Versions -timeout 30m
|
||||
func TestVersioningPaginationOver1000Versions(t *testing.T) {
|
||||
if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
|
||||
t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
|
||||
}
|
||||
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
// Create bucket
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// Enable versioning
|
||||
enableVersioning(t, client, bucketName)
|
||||
checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
|
||||
|
||||
objectKey := "test-many-versions"
|
||||
// Use 1100 for CI (tests pagination boundary), can override with env var for quick local testing
|
||||
numVersions := 1100
|
||||
if quickTest := os.Getenv("QUICK_TEST"); quickTest == "true" {
|
||||
numVersions = 50
|
||||
}
|
||||
versionIds := make([]string, 0, numVersions)
|
||||
|
||||
t.Logf("Creating %d versions of object %s...", numVersions, objectKey)
|
||||
startTime := time.Now()
|
||||
|
||||
// Create many versions of the same object
|
||||
for i := 0; i < numVersions; i++ {
|
||||
content := fmt.Sprintf("content-version-%d", i+1)
|
||||
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: strings.NewReader(content),
|
||||
})
|
||||
require.NoError(t, err, "Failed to create version %d", i+1)
|
||||
require.NotNil(t, putResp.VersionId)
|
||||
versionIds = append(versionIds, *putResp.VersionId)
|
||||
|
||||
if (i+1)%100 == 0 {
|
||||
t.Logf("Created %d/%d versions...", i+1, numVersions)
|
||||
}
|
||||
}
|
||||
|
||||
createDuration := time.Since(startTime)
|
||||
t.Logf("Created %d versions in %v", numVersions, createDuration)
|
||||
t.Logf("Version IDs collected: %d", len(versionIds))
|
||||
|
||||
// Quick sanity check - list versions to see what we have
|
||||
quickResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
t.Logf("Quick check: got %d versions, %d delete markers, truncated=%v",
|
||||
len(quickResp.Versions), len(quickResp.DeleteMarkers),
|
||||
quickResp.IsTruncated != nil && *quickResp.IsTruncated)
|
||||
|
||||
// Test 1: List all versions without pagination limit
|
||||
t.Run("ListAllVersionsNoPagination", func(t *testing.T) {
|
||||
allVersions := listAllVersions(t, client, bucketName, objectKey)
|
||||
t.Logf("Got %d versions total", len(allVersions))
|
||||
assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
|
||||
|
||||
// Verify all version IDs are unique
|
||||
versionIdSet := make(map[string]bool)
|
||||
for _, v := range allVersions {
|
||||
assert.False(t, versionIdSet[*v.VersionId], "Duplicate version ID: %s", *v.VersionId)
|
||||
versionIdSet[*v.VersionId] = true
|
||||
}
|
||||
|
||||
// Verify only one version is marked as latest
|
||||
latestCount := 0
|
||||
for _, v := range allVersions {
|
||||
if *v.IsLatest {
|
||||
latestCount++
|
||||
assert.Equal(t, versionIds[numVersions-1], *v.VersionId, "Latest should be the last created")
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 1, latestCount, "Only one version should be marked as latest")
|
||||
})
|
||||
|
||||
// Test 2: List with explicit small max-keys to force multiple pages
|
||||
t.Run("ListWithSmallMaxKeys", func(t *testing.T) {
|
||||
maxKeys := int32(100)
|
||||
allVersions := make([]types.ObjectVersion, 0, numVersions)
|
||||
var keyMarker, versionIdMarker *string
|
||||
pageCount := 0
|
||||
|
||||
for {
|
||||
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
MaxKeys: aws.Int32(maxKeys),
|
||||
KeyMarker: keyMarker,
|
||||
VersionIdMarker: versionIdMarker,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
pageCount++
|
||||
|
||||
allVersions = append(allVersions, resp.Versions...)
|
||||
|
||||
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||
break
|
||||
}
|
||||
keyMarker = resp.NextKeyMarker
|
||||
versionIdMarker = resp.NextVersionIdMarker
|
||||
}
|
||||
|
||||
t.Logf("Listed %d versions in %d pages (max-keys=%d)", len(allVersions), pageCount, maxKeys)
|
||||
assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
|
||||
if numVersions > int(maxKeys) {
|
||||
assert.Greater(t, pageCount, 1, "Should require multiple pages when versions > max-keys")
|
||||
}
|
||||
})
|
||||
|
||||
// Test 3: List with max-keys=1000 (S3 default) to test the boundary
|
||||
t.Run("ListWithDefaultMaxKeys", func(t *testing.T) {
|
||||
maxKeys := int32(1000)
|
||||
allVersions := make([]types.ObjectVersion, 0, numVersions)
|
||||
var keyMarker, versionIdMarker *string
|
||||
pageCount := 0
|
||||
|
||||
for {
|
||||
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
MaxKeys: aws.Int32(maxKeys),
|
||||
KeyMarker: keyMarker,
|
||||
VersionIdMarker: versionIdMarker,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
pageCount++
|
||||
|
||||
allVersions = append(allVersions, resp.Versions...)
|
||||
t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
|
||||
|
||||
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||
break
|
||||
}
|
||||
keyMarker = resp.NextKeyMarker
|
||||
versionIdMarker = resp.NextVersionIdMarker
|
||||
}
|
||||
|
||||
t.Logf("Listed %d versions in %d pages (max-keys=%d)", len(allVersions), pageCount, maxKeys)
|
||||
assert.Len(t, allVersions, numVersions, "Should have all %d versions", numVersions)
|
||||
if numVersions > 1000 {
|
||||
assert.GreaterOrEqual(t, pageCount, 2, "Should require at least 2 pages for >1000 versions with max-keys=1000")
|
||||
}
|
||||
})
|
||||
|
||||
// Test 4: Verify we can retrieve specific versions
|
||||
t.Run("RetrieveSpecificVersions", func(t *testing.T) {
|
||||
// Test first, middle, and last versions
|
||||
testIndices := []int{0, numVersions / 2, numVersions - 1}
|
||||
for _, idx := range testIndices {
|
||||
expectedContent := fmt.Sprintf("content-version-%d", idx+1)
|
||||
getResp, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
VersionId: aws.String(versionIds[idx]),
|
||||
})
|
||||
require.NoError(t, err, "Failed to get version at index %d", idx)
|
||||
|
||||
buf := make([]byte, len(expectedContent))
|
||||
_, err = getResp.Body.Read(buf)
|
||||
getResp.Body.Close()
|
||||
|
||||
assert.Equal(t, expectedContent, string(buf), "Content mismatch for version %d", idx+1)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestVersioningPaginationMultipleObjectsManyVersions tests pagination with multiple objects
|
||||
// each having many versions, ensuring correct handling of key-marker and version-id-marker.
|
||||
//
|
||||
// Run with: ENABLE_STRESS_TESTS=true go test -v -run TestVersioningPaginationMultipleObjectsManyVersions -timeout 30m
|
||||
func TestVersioningPaginationMultipleObjectsManyVersions(t *testing.T) {
|
||||
if os.Getenv("ENABLE_STRESS_TESTS") != "true" {
|
||||
t.Skip("Skipping stress test. Set ENABLE_STRESS_TESTS=true to run.")
|
||||
}
|
||||
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
enableVersioning(t, client, bucketName)
|
||||
checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusEnabled)
|
||||
|
||||
numObjects := 5
|
||||
versionsPerObject := 300
|
||||
totalExpectedVersions := numObjects * versionsPerObject
|
||||
|
||||
t.Logf("Creating %d objects with %d versions each (%d total)...", numObjects, versionsPerObject, totalExpectedVersions)
|
||||
startTime := time.Now()
|
||||
|
||||
// Create multiple objects with many versions each
|
||||
for i := 0; i < numObjects; i++ {
|
||||
objectKey := fmt.Sprintf("object-%03d", i)
|
||||
for j := 0; j < versionsPerObject; j++ {
|
||||
content := fmt.Sprintf("obj%d-ver%d", i, j+1)
|
||||
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(objectKey),
|
||||
Body: strings.NewReader(content),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
t.Logf("Created object %s with %d versions", objectKey, versionsPerObject)
|
||||
}
|
||||
|
||||
createDuration := time.Since(startTime)
|
||||
t.Logf("Created %d total versions in %v", totalExpectedVersions, createDuration)
|
||||
|
||||
// List all versions using pagination
|
||||
t.Run("ListAllWithPagination", func(t *testing.T) {
|
||||
maxKeys := int32(500)
|
||||
allVersions := make([]types.ObjectVersion, 0, totalExpectedVersions)
|
||||
var keyMarker, versionIdMarker *string
|
||||
pageCount := 0
|
||||
|
||||
for {
|
||||
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
MaxKeys: aws.Int32(maxKeys),
|
||||
KeyMarker: keyMarker,
|
||||
VersionIdMarker: versionIdMarker,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
pageCount++
|
||||
|
||||
allVersions = append(allVersions, resp.Versions...)
|
||||
t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
|
||||
|
||||
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||
break
|
||||
}
|
||||
keyMarker = resp.NextKeyMarker
|
||||
versionIdMarker = resp.NextVersionIdMarker
|
||||
}
|
||||
|
||||
t.Logf("Listed %d versions in %d pages", len(allVersions), pageCount)
|
||||
assert.Len(t, allVersions, totalExpectedVersions, "Should have all versions")
|
||||
|
||||
// Verify each object has correct number of versions
|
||||
versionsByKey := make(map[string]int)
|
||||
for _, v := range allVersions {
|
||||
versionsByKey[*v.Key]++
|
||||
}
|
||||
assert.Len(t, versionsByKey, numObjects, "Should have %d unique objects", numObjects)
|
||||
for key, count := range versionsByKey {
|
||||
assert.Equal(t, versionsPerObject, count, "Object %s should have %d versions", key, versionsPerObject)
|
||||
}
|
||||
|
||||
// Verify each object has exactly one latest version
|
||||
latestByKey := make(map[string]int)
|
||||
for _, v := range allVersions {
|
||||
if *v.IsLatest {
|
||||
latestByKey[*v.Key]++
|
||||
}
|
||||
}
|
||||
for key, count := range latestByKey {
|
||||
assert.Equal(t, 1, count, "Object %s should have exactly one latest version", key)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// listAllVersions is a helper to list all versions of a specific object using pagination
|
||||
func listAllVersions(t *testing.T, client *s3.Client, bucketName, objectKey string) []types.ObjectVersion {
|
||||
var allVersions []types.ObjectVersion
|
||||
var keyMarker, versionIdMarker *string
|
||||
pageCount := 0
|
||||
|
||||
for {
|
||||
resp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String(objectKey),
|
||||
KeyMarker: keyMarker,
|
||||
VersionIdMarker: versionIdMarker,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
pageCount++
|
||||
|
||||
t.Logf("Page %d: got %d versions, truncated=%v", pageCount, len(resp.Versions), resp.IsTruncated != nil && *resp.IsTruncated)
|
||||
allVersions = append(allVersions, resp.Versions...)
|
||||
|
||||
// Check if truncated (handle nil pointer)
|
||||
if resp.IsTruncated == nil || !*resp.IsTruncated {
|
||||
break
|
||||
}
|
||||
keyMarker = resp.NextKeyMarker
|
||||
versionIdMarker = resp.NextVersionIdMarker
|
||||
}
|
||||
|
||||
t.Logf("Total: %d versions in %d pages", len(allVersions), pageCount)
|
||||
return allVersions
|
||||
}
|
||||
|
||||
@@ -90,4 +90,3 @@ To debug test failures:
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
Vendored
-1
@@ -35,4 +35,3 @@
|
||||
]
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
# Makefile for TUS Protocol Integration Tests
|
||||
# This Makefile provides targets for running TUS (resumable upload) integration tests
|
||||
|
||||
# Default values
|
||||
SEAWEEDFS_BINARY ?= weed
|
||||
FILER_PORT ?= 18888
|
||||
VOLUME_PORT ?= 18080
|
||||
MASTER_PORT ?= 19333
|
||||
TEST_TIMEOUT ?= 10m
|
||||
VOLUME_MAX_SIZE_MB ?= 50
|
||||
VOLUME_MAX_COUNT ?= 100
|
||||
|
||||
# Test directory
|
||||
TEST_DIR := $(shell pwd)
|
||||
SEAWEEDFS_ROOT := $(shell cd ../.. && pwd)
|
||||
|
||||
# Colors for output
|
||||
RED := \033[0;31m
|
||||
GREEN := \033[0;32m
|
||||
YELLOW := \033[1;33m
|
||||
NC := \033[0m # No Color
|
||||
|
||||
.PHONY: all test clean start-seaweedfs stop-seaweedfs check-binary build-weed help test-basic test-chunked test-resume test-errors test-with-server
|
||||
|
||||
all: test
|
||||
|
||||
# Build SeaweedFS binary
|
||||
build-weed:
|
||||
@echo "Building SeaweedFS binary..."
|
||||
@cd $(SEAWEEDFS_ROOT)/weed && go build -o weed
|
||||
@echo "$(GREEN)SeaweedFS binary built successfully$(NC)"
|
||||
|
||||
help:
|
||||
@echo "SeaweedFS TUS Protocol Integration Tests"
|
||||
@echo ""
|
||||
@echo "Available targets:"
|
||||
@echo " test - Run all TUS integration tests"
|
||||
@echo " test-basic - Run basic TUS upload tests"
|
||||
@echo " test-chunked - Run chunked upload tests"
|
||||
@echo " test-resume - Run upload resume tests"
|
||||
@echo " test-errors - Run error handling tests"
|
||||
@echo " test-with-server - Run tests with automatic server management"
|
||||
@echo " start-seaweedfs - Start SeaweedFS server for testing"
|
||||
@echo " stop-seaweedfs - Stop SeaweedFS server"
|
||||
@echo " clean - Clean up test artifacts"
|
||||
@echo " check-binary - Check if SeaweedFS binary exists"
|
||||
@echo " build-weed - Build SeaweedFS binary"
|
||||
@echo ""
|
||||
@echo "Configuration:"
|
||||
@echo " SEAWEEDFS_BINARY=$(SEAWEEDFS_BINARY)"
|
||||
@echo " FILER_PORT=$(FILER_PORT)"
|
||||
@echo " VOLUME_PORT=$(VOLUME_PORT)"
|
||||
@echo " MASTER_PORT=$(MASTER_PORT)"
|
||||
@echo " TEST_TIMEOUT=$(TEST_TIMEOUT)"
|
||||
|
||||
check-binary:
|
||||
@if ! command -v $(SEAWEEDFS_BINARY) > /dev/null 2>&1 && [ ! -f "$(SEAWEEDFS_ROOT)/weed/weed" ]; then \
|
||||
echo "$(RED)Error: SeaweedFS binary not found$(NC)"; \
|
||||
echo "Please build SeaweedFS first: make build-weed"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "$(GREEN)SeaweedFS binary found$(NC)"
|
||||
|
||||
start-seaweedfs: check-binary
|
||||
@echo "$(YELLOW)Starting SeaweedFS server for TUS testing...$(NC)"
|
||||
@# Clean up any existing processes on our test ports
|
||||
@lsof -ti :$(MASTER_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
|
||||
@lsof -ti :$(VOLUME_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
|
||||
@lsof -ti :$(FILER_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
|
||||
@sleep 2
|
||||
|
||||
# Create necessary directories
|
||||
@mkdir -p /tmp/seaweedfs-test-tus-master
|
||||
@mkdir -p /tmp/seaweedfs-test-tus-volume
|
||||
@mkdir -p /tmp/seaweedfs-test-tus-filer
|
||||
|
||||
# Start master server (use freshly built binary)
|
||||
@echo "Starting master server..."
|
||||
@nohup $(SEAWEEDFS_ROOT)/weed/weed master \
|
||||
-port=$(MASTER_PORT) \
|
||||
-mdir=/tmp/seaweedfs-test-tus-master \
|
||||
-volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/seaweedfs-tus-master.log 2>&1 &
|
||||
@sleep 3
|
||||
|
||||
# Start volume server
|
||||
@echo "Starting volume server..."
|
||||
@nohup $(SEAWEEDFS_ROOT)/weed/weed volume \
|
||||
-port=$(VOLUME_PORT) \
|
||||
-mserver=127.0.0.1:$(MASTER_PORT) \
|
||||
-dir=/tmp/seaweedfs-test-tus-volume \
|
||||
-max=$(VOLUME_MAX_COUNT) \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/seaweedfs-tus-volume.log 2>&1 &
|
||||
@sleep 3
|
||||
|
||||
# Start filer server with TUS enabled (default tusBasePath is .tus)
|
||||
@echo "Starting filer server..."
|
||||
@nohup $(SEAWEEDFS_ROOT)/weed/weed filer \
|
||||
-port=$(FILER_PORT) \
|
||||
-master=127.0.0.1:$(MASTER_PORT) \
|
||||
-ip=127.0.0.1 \
|
||||
> /tmp/seaweedfs-tus-filer.log 2>&1 &
|
||||
@sleep 5
|
||||
|
||||
# Wait for filer to be ready
|
||||
@echo "$(YELLOW)Waiting for filer to be ready...$(NC)"
|
||||
@for i in $$(seq 1 30); do \
|
||||
if curl -s -f http://127.0.0.1:$(FILER_PORT)/ > /dev/null 2>&1; then \
|
||||
echo "$(GREEN)Filer is ready$(NC)"; \
|
||||
break; \
|
||||
fi; \
|
||||
if [ $$i -eq 30 ]; then \
|
||||
echo "$(RED)Filer failed to start within 30 seconds$(NC)"; \
|
||||
$(MAKE) debug-logs; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
echo "Waiting for filer... ($$i/30)"; \
|
||||
sleep 1; \
|
||||
done
|
||||
|
||||
@echo "$(GREEN)SeaweedFS server started successfully for TUS testing$(NC)"
|
||||
@echo "Master: http://localhost:$(MASTER_PORT)"
|
||||
@echo "Volume: http://localhost:$(VOLUME_PORT)"
|
||||
@echo "Filer: http://localhost:$(FILER_PORT)"
|
||||
@echo "TUS Endpoint: http://localhost:$(FILER_PORT)/.tus/"
|
||||
|
||||
stop-seaweedfs:
|
||||
@echo "$(YELLOW)Stopping SeaweedFS server...$(NC)"
|
||||
@lsof -ti :$(MASTER_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
|
||||
@lsof -ti :$(VOLUME_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
|
||||
@lsof -ti :$(FILER_PORT) | grep -v '^$$' | xargs kill -TERM 2>/dev/null || true
|
||||
@sleep 2
|
||||
@echo "$(GREEN)SeaweedFS server stopped$(NC)"
|
||||
|
||||
clean:
|
||||
@echo "$(YELLOW)Cleaning up TUS test artifacts...$(NC)"
|
||||
@rm -rf /tmp/seaweedfs-test-tus-*
|
||||
@rm -f /tmp/seaweedfs-tus-*.log
|
||||
@echo "$(GREEN)TUS test cleanup completed$(NC)"
|
||||
|
||||
# Run all tests
|
||||
test: check-binary
|
||||
@echo "$(YELLOW)Running all TUS integration tests...$(NC)"
|
||||
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/...
|
||||
@echo "$(GREEN)All TUS tests completed$(NC)"
|
||||
|
||||
# Run basic upload tests
|
||||
test-basic: check-binary
|
||||
@echo "$(YELLOW)Running basic TUS upload tests...$(NC)"
|
||||
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusBasicUpload|TestTusOptionsHandler" ./test/tus/...
|
||||
@echo "$(GREEN)Basic TUS tests completed$(NC)"
|
||||
|
||||
# Run chunked upload tests
|
||||
test-chunked: check-binary
|
||||
@echo "$(YELLOW)Running chunked TUS upload tests...$(NC)"
|
||||
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusChunkedUpload" ./test/tus/...
|
||||
@echo "$(GREEN)Chunked TUS tests completed$(NC)"
|
||||
|
||||
# Run resume tests
|
||||
test-resume: check-binary
|
||||
@echo "$(YELLOW)Running TUS upload resume tests...$(NC)"
|
||||
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusResumeAfterInterruption|TestTusHeadRequest" ./test/tus/...
|
||||
@echo "$(GREEN)TUS resume tests completed$(NC)"
|
||||
|
||||
# Run error handling tests
|
||||
test-errors: check-binary
|
||||
@echo "$(YELLOW)Running TUS error handling tests...$(NC)"
|
||||
@cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) -run "TestTusInvalidOffset|TestTusUploadNotFound|TestTusDeleteUpload" ./test/tus/...
|
||||
@echo "$(GREEN)TUS error tests completed$(NC)"
|
||||
|
||||
# Run tests with automatic server management
|
||||
test-with-server: build-weed
|
||||
@echo "$(YELLOW)Running TUS tests with automatic server management...$(NC)"
|
||||
@$(MAKE) -C $(TEST_DIR) start-seaweedfs && \
|
||||
sleep 3 && \
|
||||
cd $(SEAWEEDFS_ROOT) && go test -v -timeout=$(TEST_TIMEOUT) ./test/tus/...; \
|
||||
TEST_RESULT=$$?; \
|
||||
$(MAKE) -C $(TEST_DIR) stop-seaweedfs; \
|
||||
$(MAKE) -C $(TEST_DIR) clean; \
|
||||
if [ $$TEST_RESULT -eq 0 ]; then echo "$(GREEN)All TUS tests passed!$(NC)"; fi; \
|
||||
exit $$TEST_RESULT
|
||||
|
||||
# Debug targets
|
||||
debug-logs:
|
||||
@echo "$(YELLOW)=== Master Log ===$(NC)"
|
||||
@tail -n 50 /tmp/seaweedfs-tus-master.log 2>/dev/null || echo "No master log found"
|
||||
@echo "$(YELLOW)=== Volume Log ===$(NC)"
|
||||
@tail -n 50 /tmp/seaweedfs-tus-volume.log 2>/dev/null || echo "No volume log found"
|
||||
@echo "$(YELLOW)=== Filer Log ===$(NC)"
|
||||
@tail -n 50 /tmp/seaweedfs-tus-filer.log 2>/dev/null || echo "No filer log found"
|
||||
|
||||
debug-status:
|
||||
@echo "$(YELLOW)=== Process Status ===$(NC)"
|
||||
@ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
|
||||
@echo "$(YELLOW)=== Port Status ===$(NC)"
|
||||
@lsof -i :$(MASTER_PORT) -i :$(VOLUME_PORT) -i :$(FILER_PORT) 2>/dev/null || echo "No ports in use"
|
||||
|
||||
# Manual testing targets
|
||||
manual-start: start-seaweedfs
|
||||
@echo "$(GREEN)SeaweedFS is now running for manual TUS testing$(NC)"
|
||||
@echo ""
|
||||
@echo "TUS Endpoints:"
|
||||
@echo " OPTIONS /.tus/ - Capability discovery"
|
||||
@echo " POST /.tus/{path} - Create upload"
|
||||
@echo " HEAD /.tus/.uploads/{id} - Get offset"
|
||||
@echo " PATCH /.tus/.uploads/{id} - Upload data"
|
||||
@echo " DELETE /.tus/.uploads/{id} - Cancel upload"
|
||||
@echo ""
|
||||
@echo "Example curl commands:"
|
||||
@echo " curl -X OPTIONS http://localhost:$(FILER_PORT)/.tus/ -H 'Tus-Resumable: 1.0.0'"
|
||||
@echo ""
|
||||
@echo "Run 'make manual-stop' when finished"
|
||||
|
||||
manual-stop: stop-seaweedfs clean
|
||||
|
||||
# CI targets
|
||||
ci-test: test-with-server
|
||||
|
||||
# Skip integration tests (short mode)
|
||||
test-short:
|
||||
@echo "$(YELLOW)Running TUS tests in short mode (skipping integration tests)...$(NC)"
|
||||
@cd $(SEAWEEDFS_ROOT) && go test -v -short ./test/tus/...
|
||||
@echo "$(GREEN)Short tests completed$(NC)"
|
||||
|
||||
@@ -1,241 +0,0 @@
|
||||
# TUS Protocol Integration Tests
|
||||
|
||||
This directory contains integration tests for the TUS (resumable upload) protocol support in SeaweedFS Filer.
|
||||
|
||||
## Overview
|
||||
|
||||
TUS is an open protocol for resumable file uploads over HTTP. It allows clients to upload files in chunks and resume uploads after network failures or interruptions.
|
||||
|
||||
### Why TUS?
|
||||
|
||||
- **Resumable uploads**: Resume interrupted uploads without re-sending data
|
||||
- **Chunked uploads**: Upload large files in smaller pieces
|
||||
- **Simple protocol**: Standard HTTP methods with custom headers
|
||||
- **Wide client support**: Libraries available for JavaScript, Python, Go, and more
|
||||
|
||||
## TUS Protocol Endpoints
|
||||
|
||||
| Method | Path | Description |
|
||||
|--------|------|-------------|
|
||||
| `OPTIONS` | `/.tus/` | Server capability discovery |
|
||||
| `POST` | `/.tus/{path}` | Create new upload session |
|
||||
| `HEAD` | `/.tus/.uploads/{id}` | Get current upload offset |
|
||||
| `PATCH` | `/.tus/.uploads/{id}` | Upload data at offset |
|
||||
| `DELETE` | `/.tus/.uploads/{id}` | Cancel upload |
|
||||
|
||||
### TUS Headers
|
||||
|
||||
**Request Headers:**
|
||||
- `Tus-Resumable: 1.0.0` - Protocol version (required)
|
||||
- `Upload-Length` - Total file size in bytes (required on POST)
|
||||
- `Upload-Offset` - Current byte offset (required on PATCH)
|
||||
- `Upload-Metadata` - Base64-encoded key-value pairs (optional)
|
||||
- `Content-Type: application/offset+octet-stream` (required on PATCH)
|
||||
|
||||
**Response Headers:**
|
||||
- `Tus-Resumable` - Protocol version
|
||||
- `Tus-Version` - Supported versions
|
||||
- `Tus-Extension` - Supported extensions
|
||||
- `Tus-Max-Size` - Maximum upload size
|
||||
- `Upload-Offset` - Current byte offset
|
||||
- `Location` - Upload URL (on POST)
|
||||
|
||||
## Enabling TUS
|
||||
|
||||
TUS protocol support is enabled by default at `/.tus` path. You can customize the path using the `-tusBasePath` flag:
|
||||
|
||||
```bash
|
||||
# Start filer with default TUS path (/.tus)
|
||||
weed filer -master=localhost:9333
|
||||
|
||||
# Use a custom path (leading slash added automatically if missing)
|
||||
weed filer -master=localhost:9333 -tusBasePath=/.uploads/tus
|
||||
|
||||
# Disable TUS by setting empty path
|
||||
weed filer -master=localhost:9333 -tusBasePath=
|
||||
```
|
||||
|
||||
## Test Structure
|
||||
|
||||
### Integration Tests
|
||||
|
||||
The tests cover:
|
||||
|
||||
1. **Basic Functionality**
|
||||
- `TestTusOptionsHandler` - Capability discovery
|
||||
- `TestTusBasicUpload` - Simple complete upload
|
||||
- `TestTusCreationWithUpload` - Creation-with-upload extension
|
||||
|
||||
2. **Chunked Uploads**
|
||||
- `TestTusChunkedUpload` - Upload in multiple chunks
|
||||
|
||||
3. **Resumable Uploads**
|
||||
- `TestTusHeadRequest` - Offset tracking
|
||||
- `TestTusResumeAfterInterruption` - Resume after failure
|
||||
|
||||
4. **Error Handling**
|
||||
- `TestTusInvalidOffset` - Offset mismatch (409 Conflict)
|
||||
- `TestTusUploadNotFound` - Missing upload (404 Not Found)
|
||||
- `TestTusDeleteUpload` - Upload cancellation
|
||||
|
||||
## Running Tests
|
||||
|
||||
### Prerequisites
|
||||
|
||||
1. **Build SeaweedFS**:
|
||||
```bash
|
||||
make build-weed
|
||||
# or
|
||||
cd ../../weed && go build -o weed
|
||||
```
|
||||
|
||||
### Using Makefile
|
||||
|
||||
```bash
|
||||
# Show available targets
|
||||
make help
|
||||
|
||||
# Run all tests with automatic server management
|
||||
make test-with-server
|
||||
|
||||
# Run all tests (requires running server)
|
||||
make test
|
||||
|
||||
# Run specific test categories
|
||||
make test-basic # Basic upload tests
|
||||
make test-chunked # Chunked upload tests
|
||||
make test-resume # Resume/HEAD tests
|
||||
make test-errors # Error handling tests
|
||||
|
||||
# Manual testing
|
||||
make manual-start # Start SeaweedFS for manual testing
|
||||
make manual-stop # Stop and cleanup
|
||||
```
|
||||
|
||||
### Using Go Test Directly
|
||||
|
||||
```bash
|
||||
# Run all TUS tests
|
||||
go test -v ./test/tus/...
|
||||
|
||||
# Run specific test
|
||||
go test -v ./test/tus -run TestTusBasicUpload
|
||||
|
||||
# Skip integration tests (short mode)
|
||||
go test -v -short ./test/tus/...
|
||||
```
|
||||
|
||||
### Debug
|
||||
|
||||
```bash
|
||||
# View server logs
|
||||
make debug-logs
|
||||
|
||||
# Check process and port status
|
||||
make debug-status
|
||||
```
|
||||
|
||||
## Test Environment
|
||||
|
||||
Each test run:
|
||||
1. Starts a SeaweedFS cluster (master, volume, filer)
|
||||
2. Creates uploads using TUS protocol
|
||||
3. Verifies files are stored correctly
|
||||
4. Cleans up test data
|
||||
|
||||
### Default Ports
|
||||
|
||||
| Service | Port |
|
||||
|---------|------|
|
||||
| Master | 19333 |
|
||||
| Volume | 18080 |
|
||||
| Filer | 18888 |
|
||||
|
||||
### Configuration
|
||||
|
||||
Override defaults via environment or Makefile variables:
|
||||
```bash
|
||||
FILER_PORT=8889 MASTER_PORT=9334 make test
|
||||
```
|
||||
|
||||
## Example Usage
|
||||
|
||||
### Create Upload
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:18888/.tus/mydir/file.txt \
|
||||
-H "Tus-Resumable: 1.0.0" \
|
||||
-H "Upload-Length: 1000" \
|
||||
-H "Upload-Metadata: filename dGVzdC50eHQ="
|
||||
```
|
||||
|
||||
### Upload Data
|
||||
|
||||
```bash
|
||||
curl -X PATCH http://localhost:18888/.tus/.uploads/{upload-id} \
|
||||
-H "Tus-Resumable: 1.0.0" \
|
||||
-H "Upload-Offset: 0" \
|
||||
-H "Content-Type: application/offset+octet-stream" \
|
||||
--data-binary @file.txt
|
||||
```
|
||||
|
||||
### Check Offset
|
||||
|
||||
```bash
|
||||
curl -I http://localhost:18888/.tus/.uploads/{upload-id} \
|
||||
-H "Tus-Resumable: 1.0.0"
|
||||
```
|
||||
|
||||
### Cancel Upload
|
||||
|
||||
```bash
|
||||
curl -X DELETE http://localhost:18888/.tus/.uploads/{upload-id} \
|
||||
-H "Tus-Resumable: 1.0.0"
|
||||
```
|
||||
|
||||
## TUS Extensions Supported
|
||||
|
||||
- **creation**: Create new uploads with POST
|
||||
- **creation-with-upload**: Send data in creation request
|
||||
- **termination**: Cancel uploads with DELETE
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
Client Filer Volume Servers
|
||||
| | |
|
||||
|-- POST /.tus/path/file.mp4 ->| |
|
||||
| |-- Create session dir ------->|
|
||||
|<-- 201 Location: /.../{id} --| |
|
||||
| | |
|
||||
|-- PATCH /.tus/.uploads/{id} >| |
|
||||
| Upload-Offset: 0 |-- Assign volume ------------>|
|
||||
| [chunk data] |-- Upload chunk ------------->|
|
||||
|<-- 204 Upload-Offset: N -----| |
|
||||
| | |
|
||||
| (network failure) | |
|
||||
| | |
|
||||
|-- HEAD /.tus/.uploads/{id} ->| |
|
||||
|<-- Upload-Offset: N ---------| |
|
||||
| | |
|
||||
|-- PATCH (resume) ----------->|-- Upload remaining -------->|
|
||||
|<-- 204 (complete) -----------|-- Assemble final file ----->|
|
||||
```
|
||||
|
||||
## Comparison with S3 Multipart
|
||||
|
||||
| Feature | TUS | S3 Multipart |
|
||||
|---------|-----|--------------|
|
||||
| Protocol | Custom HTTP headers | S3 API |
|
||||
| Session Init | POST with Upload-Length | CreateMultipartUpload |
|
||||
| Upload Data | PATCH with offset | UploadPart with partNumber |
|
||||
| Resume | HEAD to get offset | ListParts |
|
||||
| Complete | Automatic at final offset | CompleteMultipartUpload |
|
||||
| Ordering | Sequential (offset-based) | Parallel (part numbers) |
|
||||
|
||||
## Related Resources
|
||||
|
||||
- [TUS Protocol Specification](https://tus.io/protocols/resumable-upload)
|
||||
- [tus-js-client](https://github.com/tus/tus-js-client) - JavaScript client
|
||||
- [go-tus](https://github.com/eventials/go-tus) - Go client
|
||||
- [SeaweedFS S3 API](../../weed/s3api) - Alternative multipart upload
|
||||
@@ -1,775 +0,0 @@
|
||||
package tus
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
TusVersion = "1.0.0"
|
||||
testFilerPort = "18888"
|
||||
testMasterPort = "19333"
|
||||
testVolumePort = "18080"
|
||||
)
|
||||
|
||||
// TestCluster represents a running SeaweedFS cluster for testing
|
||||
type TestCluster struct {
|
||||
masterCmd *exec.Cmd
|
||||
volumeCmd *exec.Cmd
|
||||
filerCmd *exec.Cmd
|
||||
dataDir string
|
||||
}
|
||||
|
||||
func (c *TestCluster) Stop() {
|
||||
if c.filerCmd != nil && c.filerCmd.Process != nil {
|
||||
c.filerCmd.Process.Signal(os.Interrupt)
|
||||
c.filerCmd.Wait()
|
||||
}
|
||||
if c.volumeCmd != nil && c.volumeCmd.Process != nil {
|
||||
c.volumeCmd.Process.Signal(os.Interrupt)
|
||||
c.volumeCmd.Wait()
|
||||
}
|
||||
if c.masterCmd != nil && c.masterCmd.Process != nil {
|
||||
c.masterCmd.Process.Signal(os.Interrupt)
|
||||
c.masterCmd.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *TestCluster) FilerURL() string {
|
||||
return fmt.Sprintf("http://127.0.0.1:%s", testFilerPort)
|
||||
}
|
||||
|
||||
func (c *TestCluster) TusURL() string {
|
||||
return fmt.Sprintf("%s/.tus", c.FilerURL())
|
||||
}
|
||||
|
||||
// FullURL converts a relative path to a full URL
|
||||
func (c *TestCluster) FullURL(path string) string {
|
||||
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
|
||||
return path
|
||||
}
|
||||
return fmt.Sprintf("http://127.0.0.1:%s%s", testFilerPort, path)
|
||||
}
|
||||
|
||||
// startTestCluster starts a SeaweedFS cluster for testing
|
||||
func startTestCluster(t *testing.T, ctx context.Context) (*TestCluster, error) {
|
||||
weedBinary := findWeedBinary()
|
||||
if weedBinary == "" {
|
||||
return nil, fmt.Errorf("weed binary not found - please build it first: cd weed && go build")
|
||||
}
|
||||
|
||||
dataDir, err := os.MkdirTemp("", "seaweedfs_tus_test_")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cluster := &TestCluster{dataDir: dataDir}
|
||||
|
||||
// Create subdirectories
|
||||
masterDir := filepath.Join(dataDir, "master")
|
||||
volumeDir := filepath.Join(dataDir, "volume")
|
||||
filerDir := filepath.Join(dataDir, "filer")
|
||||
os.MkdirAll(masterDir, 0755)
|
||||
os.MkdirAll(volumeDir, 0755)
|
||||
os.MkdirAll(filerDir, 0755)
|
||||
|
||||
// Start master
|
||||
masterCmd := exec.CommandContext(ctx, weedBinary, "master",
|
||||
"-port", testMasterPort,
|
||||
"-mdir", masterDir,
|
||||
"-ip", "127.0.0.1",
|
||||
)
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
if err != nil {
|
||||
os.RemoveAll(dataDir)
|
||||
return nil, fmt.Errorf("failed to create master log: %v", err)
|
||||
}
|
||||
masterCmd.Stdout = masterLogFile
|
||||
masterCmd.Stderr = masterLogFile
|
||||
if err := masterCmd.Start(); err != nil {
|
||||
os.RemoveAll(dataDir)
|
||||
return nil, fmt.Errorf("failed to start master: %v", err)
|
||||
}
|
||||
cluster.masterCmd = masterCmd
|
||||
|
||||
// Wait for master to be ready
|
||||
if err := waitForHTTPServer("http://127.0.0.1:"+testMasterPort+"/dir/status", 30*time.Second); err != nil {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(dataDir)
|
||||
return nil, fmt.Errorf("master not ready: %v", err)
|
||||
}
|
||||
|
||||
// Start volume server
|
||||
volumeCmd := exec.CommandContext(ctx, weedBinary, "volume",
|
||||
"-port", testVolumePort,
|
||||
"-dir", volumeDir,
|
||||
"-mserver", "127.0.0.1:"+testMasterPort,
|
||||
"-ip", "127.0.0.1",
|
||||
)
|
||||
volumeLogFile, err := os.Create(filepath.Join(volumeDir, "volume.log"))
|
||||
if err != nil {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(dataDir)
|
||||
return nil, fmt.Errorf("failed to create volume log: %v", err)
|
||||
}
|
||||
volumeCmd.Stdout = volumeLogFile
|
||||
volumeCmd.Stderr = volumeLogFile
|
||||
if err := volumeCmd.Start(); err != nil {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(dataDir)
|
||||
return nil, fmt.Errorf("failed to start volume server: %v", err)
|
||||
}
|
||||
cluster.volumeCmd = volumeCmd
|
||||
|
||||
// Wait for volume server to register with master
|
||||
if err := waitForHTTPServer("http://127.0.0.1:"+testVolumePort+"/status", 30*time.Second); err != nil {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(dataDir)
|
||||
return nil, fmt.Errorf("volume server not ready: %v", err)
|
||||
}
|
||||
|
||||
// Start filer with TUS enabled
|
||||
filerCmd := exec.CommandContext(ctx, weedBinary, "filer",
|
||||
"-port", testFilerPort,
|
||||
"-master", "127.0.0.1:"+testMasterPort,
|
||||
"-ip", "127.0.0.1",
|
||||
"-defaultStoreDir", filerDir,
|
||||
"-tusBasePath", "/.tus",
|
||||
)
|
||||
filerLogFile, err := os.Create(filepath.Join(filerDir, "filer.log"))
|
||||
if err != nil {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(dataDir)
|
||||
return nil, fmt.Errorf("failed to create filer log: %v", err)
|
||||
}
|
||||
filerCmd.Stdout = filerLogFile
|
||||
filerCmd.Stderr = filerLogFile
|
||||
if err := filerCmd.Start(); err != nil {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(dataDir)
|
||||
return nil, fmt.Errorf("failed to start filer: %v", err)
|
||||
}
|
||||
cluster.filerCmd = filerCmd
|
||||
|
||||
// Wait for filer
|
||||
if err := waitForHTTPServer("http://127.0.0.1:"+testFilerPort+"/", 30*time.Second); err != nil {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(dataDir)
|
||||
return nil, fmt.Errorf("filer not ready: %v", err)
|
||||
}
|
||||
|
||||
// Wait a bit more for the cluster to fully stabilize
|
||||
// Volumes are created lazily, and we need to ensure the master topology is ready
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
return cluster, nil
|
||||
}
|
||||
|
||||
func findWeedBinary() string {
|
||||
candidates := []string{
|
||||
"../../weed/weed",
|
||||
"../weed/weed",
|
||||
"./weed/weed",
|
||||
"weed",
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
if path, err := exec.LookPath("weed"); err == nil {
|
||||
return path
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func waitForHTTPServer(url string, timeout time.Duration) error {
|
||||
start := time.Now()
|
||||
client := &http.Client{Timeout: 1 * time.Second}
|
||||
for time.Since(start) < timeout {
|
||||
resp, err := client.Get(url)
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
return nil
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("timeout waiting for %s", url)
|
||||
}
|
||||
|
||||
// encodeTusMetadata encodes key-value pairs for Upload-Metadata header
|
||||
func encodeTusMetadata(metadata map[string]string) string {
|
||||
var parts []string
|
||||
for k, v := range metadata {
|
||||
encoded := base64.StdEncoding.EncodeToString([]byte(v))
|
||||
parts = append(parts, fmt.Sprintf("%s %s", k, encoded))
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
// TestTusOptionsHandler tests the OPTIONS endpoint for capability discovery
|
||||
func TestTusOptionsHandler(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startTestCluster(t, ctx)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(cluster.dataDir)
|
||||
}()
|
||||
|
||||
// Test OPTIONS request
|
||||
req, err := http.NewRequest(http.MethodOptions, cluster.TusURL()+"/", nil)
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Tus-Resumable", TusVersion)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Verify TUS headers
|
||||
assert.Equal(t, http.StatusOK, resp.StatusCode, "OPTIONS should return 200 OK")
|
||||
assert.Equal(t, TusVersion, resp.Header.Get("Tus-Resumable"), "Should return Tus-Resumable header")
|
||||
assert.NotEmpty(t, resp.Header.Get("Tus-Version"), "Should return Tus-Version header")
|
||||
assert.NotEmpty(t, resp.Header.Get("Tus-Extension"), "Should return Tus-Extension header")
|
||||
assert.NotEmpty(t, resp.Header.Get("Tus-Max-Size"), "Should return Tus-Max-Size header")
|
||||
}
|
||||
|
||||
// TestTusBasicUpload tests a simple complete upload
|
||||
func TestTusBasicUpload(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startTestCluster(t, ctx)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(cluster.dataDir)
|
||||
}()
|
||||
|
||||
testData := []byte("Hello, TUS Protocol! This is a test file.")
|
||||
targetPath := "/testdir/testfile.txt"
|
||||
|
||||
// Step 1: Create upload (POST)
|
||||
createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
|
||||
require.NoError(t, err)
|
||||
createReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
|
||||
createReq.Header.Set("Upload-Metadata", encodeTusMetadata(map[string]string{
|
||||
"filename": "testfile.txt",
|
||||
"content-type": "text/plain",
|
||||
}))
|
||||
|
||||
client := &http.Client{}
|
||||
createResp, err := client.Do(createReq)
|
||||
require.NoError(t, err)
|
||||
defer createResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusCreated, createResp.StatusCode, "POST should return 201 Created")
|
||||
uploadLocation := createResp.Header.Get("Location")
|
||||
assert.NotEmpty(t, uploadLocation, "Should return Location header with upload URL")
|
||||
t.Logf("Upload location: %s", uploadLocation)
|
||||
|
||||
// Step 2: Upload data (PATCH)
|
||||
patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData))
|
||||
require.NoError(t, err)
|
||||
patchReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
patchReq.Header.Set("Upload-Offset", "0")
|
||||
patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
|
||||
patchReq.Header.Set("Content-Length", strconv.Itoa(len(testData)))
|
||||
|
||||
patchResp, err := client.Do(patchReq)
|
||||
require.NoError(t, err)
|
||||
defer patchResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusNoContent, patchResp.StatusCode, "PATCH should return 204 No Content")
|
||||
newOffset := patchResp.Header.Get("Upload-Offset")
|
||||
assert.Equal(t, strconv.Itoa(len(testData)), newOffset, "Upload-Offset should equal total file size")
|
||||
|
||||
// Step 3: Verify the file was created
|
||||
getResp, err := client.Get(cluster.FilerURL() + targetPath)
|
||||
require.NoError(t, err)
|
||||
defer getResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusOK, getResp.StatusCode, "GET should return 200 OK")
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testData, body, "File content should match uploaded data")
|
||||
}
|
||||
|
||||
// TestTusChunkedUpload tests uploading a file in multiple chunks
|
||||
func TestTusChunkedUpload(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startTestCluster(t, ctx)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(cluster.dataDir)
|
||||
}()
|
||||
|
||||
// Create test data (100KB)
|
||||
testData := make([]byte, 100*1024)
|
||||
for i := range testData {
|
||||
testData[i] = byte(i % 256)
|
||||
}
|
||||
chunkSize := 32 * 1024 // 32KB chunks
|
||||
targetPath := "/chunked/largefile.bin"
|
||||
|
||||
client := &http.Client{}
|
||||
|
||||
// Step 1: Create upload
|
||||
createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
|
||||
require.NoError(t, err)
|
||||
createReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
|
||||
|
||||
createResp, err := client.Do(createReq)
|
||||
require.NoError(t, err)
|
||||
defer createResp.Body.Close()
|
||||
|
||||
require.Equal(t, http.StatusCreated, createResp.StatusCode)
|
||||
uploadLocation := createResp.Header.Get("Location")
|
||||
require.NotEmpty(t, uploadLocation)
|
||||
t.Logf("Upload location: %s", uploadLocation)
|
||||
|
||||
// Step 2: Upload in chunks
|
||||
offset := 0
|
||||
for offset < len(testData) {
|
||||
end := offset + chunkSize
|
||||
if end > len(testData) {
|
||||
end = len(testData)
|
||||
}
|
||||
chunk := testData[offset:end]
|
||||
|
||||
patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(chunk))
|
||||
require.NoError(t, err)
|
||||
patchReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
patchReq.Header.Set("Upload-Offset", strconv.Itoa(offset))
|
||||
patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
|
||||
patchReq.Header.Set("Content-Length", strconv.Itoa(len(chunk)))
|
||||
|
||||
patchResp, err := client.Do(patchReq)
|
||||
require.NoError(t, err)
|
||||
patchResp.Body.Close()
|
||||
|
||||
require.Equal(t, http.StatusNoContent, patchResp.StatusCode,
|
||||
"PATCH chunk at offset %d should return 204", offset)
|
||||
newOffset, err := strconv.Atoi(patchResp.Header.Get("Upload-Offset"))
|
||||
require.NoError(t, err, "Upload-Offset header should be a valid integer")
|
||||
require.Equal(t, end, newOffset, "New offset should be %d", end)
|
||||
|
||||
t.Logf("Uploaded chunk: offset=%d, size=%d, newOffset=%d", offset, len(chunk), newOffset)
|
||||
offset = end
|
||||
}
|
||||
|
||||
// Step 3: Verify the complete file
|
||||
getResp, err := client.Get(cluster.FilerURL() + targetPath)
|
||||
require.NoError(t, err)
|
||||
defer getResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusOK, getResp.StatusCode)
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testData, body, "File content should match uploaded data")
|
||||
}
|
||||
|
||||
// TestTusHeadRequest tests the HEAD endpoint to get upload offset
|
||||
func TestTusHeadRequest(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startTestCluster(t, ctx)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(cluster.dataDir)
|
||||
}()
|
||||
|
||||
testData := []byte("Test data for HEAD request verification")
|
||||
targetPath := "/headtest/file.txt"
|
||||
client := &http.Client{}
|
||||
|
||||
// Create upload
|
||||
createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
|
||||
require.NoError(t, err)
|
||||
createReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
|
||||
|
||||
createResp, err := client.Do(createReq)
|
||||
require.NoError(t, err)
|
||||
defer createResp.Body.Close()
|
||||
require.Equal(t, http.StatusCreated, createResp.StatusCode)
|
||||
uploadLocation := createResp.Header.Get("Location")
|
||||
|
||||
// HEAD before any data uploaded - offset should be 0
|
||||
headReq1, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil)
|
||||
require.NoError(t, err)
|
||||
headReq1.Header.Set("Tus-Resumable", TusVersion)
|
||||
|
||||
headResp1, err := client.Do(headReq1)
|
||||
require.NoError(t, err)
|
||||
defer headResp1.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusOK, headResp1.StatusCode)
|
||||
assert.Equal(t, "0", headResp1.Header.Get("Upload-Offset"), "Initial offset should be 0")
|
||||
assert.Equal(t, strconv.Itoa(len(testData)), headResp1.Header.Get("Upload-Length"))
|
||||
|
||||
// Upload half the data
|
||||
halfLen := len(testData) / 2
|
||||
patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[:halfLen]))
|
||||
require.NoError(t, err)
|
||||
patchReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
patchReq.Header.Set("Upload-Offset", "0")
|
||||
patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
|
||||
|
||||
patchResp, err := client.Do(patchReq)
|
||||
require.NoError(t, err)
|
||||
patchResp.Body.Close()
|
||||
require.Equal(t, http.StatusNoContent, patchResp.StatusCode)
|
||||
|
||||
// HEAD after partial upload - offset should be halfLen
|
||||
headReq2, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil)
|
||||
require.NoError(t, err)
|
||||
headReq2.Header.Set("Tus-Resumable", TusVersion)
|
||||
|
||||
headResp2, err := client.Do(headReq2)
|
||||
require.NoError(t, err)
|
||||
defer headResp2.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusOK, headResp2.StatusCode)
|
||||
assert.Equal(t, strconv.Itoa(halfLen), headResp2.Header.Get("Upload-Offset"),
|
||||
"Offset should be %d after partial upload", halfLen)
|
||||
}
|
||||
|
||||
// TestTusDeleteUpload tests canceling an in-progress upload
|
||||
func TestTusDeleteUpload(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startTestCluster(t, ctx)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(cluster.dataDir)
|
||||
}()
|
||||
|
||||
testData := []byte("Data to be deleted")
|
||||
targetPath := "/deletetest/file.txt"
|
||||
client := &http.Client{}
|
||||
|
||||
// Create upload
|
||||
createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
|
||||
require.NoError(t, err)
|
||||
createReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
|
||||
|
||||
createResp, err := client.Do(createReq)
|
||||
require.NoError(t, err)
|
||||
defer createResp.Body.Close()
|
||||
require.Equal(t, http.StatusCreated, createResp.StatusCode)
|
||||
uploadLocation := createResp.Header.Get("Location")
|
||||
|
||||
// Upload some data
|
||||
patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[:10]))
|
||||
require.NoError(t, err)
|
||||
patchReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
patchReq.Header.Set("Upload-Offset", "0")
|
||||
patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
|
||||
|
||||
patchResp, err := client.Do(patchReq)
|
||||
require.NoError(t, err)
|
||||
patchResp.Body.Close()
|
||||
|
||||
// Delete the upload
|
||||
deleteReq, err := http.NewRequest(http.MethodDelete, cluster.FullURL(uploadLocation), nil)
|
||||
require.NoError(t, err)
|
||||
deleteReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
|
||||
deleteResp, err := client.Do(deleteReq)
|
||||
require.NoError(t, err)
|
||||
defer deleteResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusNoContent, deleteResp.StatusCode, "DELETE should return 204")
|
||||
|
||||
// Verify upload is gone - HEAD should return 404
|
||||
headReq, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil)
|
||||
require.NoError(t, err)
|
||||
headReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
|
||||
headResp, err := client.Do(headReq)
|
||||
require.NoError(t, err)
|
||||
defer headResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, headResp.StatusCode, "HEAD after DELETE should return 404")
|
||||
}
|
||||
|
||||
// TestTusInvalidOffset tests error handling for mismatched offsets
|
||||
func TestTusInvalidOffset(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startTestCluster(t, ctx)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(cluster.dataDir)
|
||||
}()
|
||||
|
||||
testData := []byte("Test data for offset validation")
|
||||
targetPath := "/offsettest/file.txt"
|
||||
client := &http.Client{}
|
||||
|
||||
// Create upload
|
||||
createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
|
||||
require.NoError(t, err)
|
||||
createReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
|
||||
|
||||
createResp, err := client.Do(createReq)
|
||||
require.NoError(t, err)
|
||||
defer createResp.Body.Close()
|
||||
require.Equal(t, http.StatusCreated, createResp.StatusCode)
|
||||
uploadLocation := createResp.Header.Get("Location")
|
||||
|
||||
// Try to upload with wrong offset (should be 0, but we send 100)
|
||||
patchReq, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData))
|
||||
require.NoError(t, err)
|
||||
patchReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
patchReq.Header.Set("Upload-Offset", "100") // Wrong offset!
|
||||
patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
|
||||
|
||||
patchResp, err := client.Do(patchReq)
|
||||
require.NoError(t, err)
|
||||
defer patchResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusConflict, patchResp.StatusCode,
|
||||
"PATCH with wrong offset should return 409 Conflict")
|
||||
}
|
||||
|
||||
// TestTusUploadNotFound tests accessing a non-existent upload
|
||||
func TestTusUploadNotFound(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startTestCluster(t, ctx)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(cluster.dataDir)
|
||||
}()
|
||||
|
||||
client := &http.Client{}
|
||||
fakeUploadURL := cluster.TusURL() + "/.uploads/nonexistent-upload-id"
|
||||
|
||||
// HEAD on non-existent upload
|
||||
headReq, err := http.NewRequest(http.MethodHead, fakeUploadURL, nil)
|
||||
require.NoError(t, err)
|
||||
headReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
|
||||
headResp, err := client.Do(headReq)
|
||||
require.NoError(t, err)
|
||||
defer headResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, headResp.StatusCode,
|
||||
"HEAD on non-existent upload should return 404")
|
||||
|
||||
// PATCH on non-existent upload
|
||||
patchReq, err := http.NewRequest(http.MethodPatch, fakeUploadURL, bytes.NewReader([]byte("data")))
|
||||
require.NoError(t, err)
|
||||
patchReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
patchReq.Header.Set("Upload-Offset", "0")
|
||||
patchReq.Header.Set("Content-Type", "application/offset+octet-stream")
|
||||
|
||||
patchResp, err := client.Do(patchReq)
|
||||
require.NoError(t, err)
|
||||
defer patchResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, patchResp.StatusCode,
|
||||
"PATCH on non-existent upload should return 404")
|
||||
}
|
||||
|
||||
// TestTusCreationWithUpload tests the creation-with-upload extension
|
||||
func TestTusCreationWithUpload(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startTestCluster(t, ctx)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(cluster.dataDir)
|
||||
}()
|
||||
|
||||
testData := []byte("Small file uploaded in creation request")
|
||||
targetPath := "/creationwithupload/smallfile.txt"
|
||||
client := &http.Client{}
|
||||
|
||||
// Create upload with data in the same request
|
||||
createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, bytes.NewReader(testData))
|
||||
require.NoError(t, err)
|
||||
createReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
|
||||
createReq.Header.Set("Content-Type", "application/offset+octet-stream")
|
||||
|
||||
createResp, err := client.Do(createReq)
|
||||
require.NoError(t, err)
|
||||
defer createResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusCreated, createResp.StatusCode)
|
||||
uploadLocation := createResp.Header.Get("Location")
|
||||
assert.NotEmpty(t, uploadLocation)
|
||||
|
||||
// Check Upload-Offset header - should indicate all data was received
|
||||
uploadOffset := createResp.Header.Get("Upload-Offset")
|
||||
assert.Equal(t, strconv.Itoa(len(testData)), uploadOffset,
|
||||
"Upload-Offset should equal file size for complete upload")
|
||||
|
||||
// Verify the file
|
||||
getResp, err := client.Get(cluster.FilerURL() + targetPath)
|
||||
require.NoError(t, err)
|
||||
defer getResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusOK, getResp.StatusCode)
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testData, body)
|
||||
}
|
||||
|
||||
// TestTusResumeAfterInterruption simulates resuming an upload after failure
|
||||
func TestTusResumeAfterInterruption(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startTestCluster(t, ctx)
|
||||
require.NoError(t, err)
|
||||
defer func() {
|
||||
cluster.Stop()
|
||||
os.RemoveAll(cluster.dataDir)
|
||||
}()
|
||||
|
||||
// 50KB test data
|
||||
testData := make([]byte, 50*1024)
|
||||
for i := range testData {
|
||||
testData[i] = byte(i % 256)
|
||||
}
|
||||
targetPath := "/resume/interrupted.bin"
|
||||
client := &http.Client{}
|
||||
|
||||
// Create upload
|
||||
createReq, err := http.NewRequest(http.MethodPost, cluster.TusURL()+targetPath, nil)
|
||||
require.NoError(t, err)
|
||||
createReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
createReq.Header.Set("Upload-Length", strconv.Itoa(len(testData)))
|
||||
|
||||
createResp, err := client.Do(createReq)
|
||||
require.NoError(t, err)
|
||||
defer createResp.Body.Close()
|
||||
require.Equal(t, http.StatusCreated, createResp.StatusCode)
|
||||
uploadLocation := createResp.Header.Get("Location")
|
||||
|
||||
// Upload first 20KB
|
||||
firstChunkSize := 20 * 1024
|
||||
patchReq1, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[:firstChunkSize]))
|
||||
require.NoError(t, err)
|
||||
patchReq1.Header.Set("Tus-Resumable", TusVersion)
|
||||
patchReq1.Header.Set("Upload-Offset", "0")
|
||||
patchReq1.Header.Set("Content-Type", "application/offset+octet-stream")
|
||||
|
||||
patchResp1, err := client.Do(patchReq1)
|
||||
require.NoError(t, err)
|
||||
patchResp1.Body.Close()
|
||||
require.Equal(t, http.StatusNoContent, patchResp1.StatusCode)
|
||||
|
||||
t.Log("Simulating network interruption...")
|
||||
|
||||
// Simulate resumption: Query current offset with HEAD
|
||||
headReq, err := http.NewRequest(http.MethodHead, cluster.FullURL(uploadLocation), nil)
|
||||
require.NoError(t, err)
|
||||
headReq.Header.Set("Tus-Resumable", TusVersion)
|
||||
|
||||
headResp, err := client.Do(headReq)
|
||||
require.NoError(t, err)
|
||||
defer headResp.Body.Close()
|
||||
|
||||
require.Equal(t, http.StatusOK, headResp.StatusCode)
|
||||
currentOffset, err := strconv.Atoi(headResp.Header.Get("Upload-Offset"))
|
||||
require.NoError(t, err, "Upload-Offset header should be a valid integer")
|
||||
t.Logf("Resumed upload at offset: %d", currentOffset)
|
||||
require.Equal(t, firstChunkSize, currentOffset)
|
||||
|
||||
// Resume upload from current offset
|
||||
patchReq2, err := http.NewRequest(http.MethodPatch, cluster.FullURL(uploadLocation), bytes.NewReader(testData[currentOffset:]))
|
||||
require.NoError(t, err)
|
||||
patchReq2.Header.Set("Tus-Resumable", TusVersion)
|
||||
patchReq2.Header.Set("Upload-Offset", strconv.Itoa(currentOffset))
|
||||
patchReq2.Header.Set("Content-Type", "application/offset+octet-stream")
|
||||
|
||||
patchResp2, err := client.Do(patchReq2)
|
||||
require.NoError(t, err)
|
||||
patchResp2.Body.Close()
|
||||
require.Equal(t, http.StatusNoContent, patchResp2.StatusCode)
|
||||
|
||||
// Verify complete file
|
||||
getResp, err := client.Get(cluster.FilerURL() + targetPath)
|
||||
require.NoError(t, err)
|
||||
defer getResp.Body.Close()
|
||||
|
||||
assert.Equal(t, http.StatusOK, getResp.StatusCode)
|
||||
body, err := io.ReadAll(getResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testData, body, "Resumed upload should produce complete file")
|
||||
}
|
||||
@@ -109,6 +109,13 @@ func (s *AdminServer) GetAdminData(username string) (AdminData, error) {
|
||||
glog.Errorf("Failed to get cluster volume servers: %v", err)
|
||||
return AdminData{}, err
|
||||
}
|
||||
// Sort the servers so they show up in consistent order after each reload
|
||||
sort.Slice(volumeServersData.VolumeServers, func(i, j int) bool {
|
||||
s1Name := volumeServersData.VolumeServers[i].GetDisplayAddress()
|
||||
s2Name := volumeServersData.VolumeServers[j].GetDisplayAddress()
|
||||
|
||||
return s1Name < s2Name
|
||||
})
|
||||
|
||||
// Get master nodes status
|
||||
masterNodes := s.getMasterNodesStatus()
|
||||
@@ -258,11 +265,6 @@ func (s *AdminServer) getFilerNodesStatus() []FilerNode {
|
||||
return []FilerNode{}
|
||||
}
|
||||
|
||||
// Sort filer nodes by address for consistent ordering on page refresh
|
||||
sort.Slice(filerNodes, func(i, j int) bool {
|
||||
return filerNodes[i].Address < filerNodes[j].Address
|
||||
})
|
||||
|
||||
return filerNodes
|
||||
}
|
||||
|
||||
@@ -299,10 +301,5 @@ func (s *AdminServer) getMessageBrokerNodesStatus() []MessageBrokerNode {
|
||||
return []MessageBrokerNode{}
|
||||
}
|
||||
|
||||
// Sort message broker nodes by address for consistent ordering on page refresh
|
||||
sort.Slice(messageBrokers, func(i, j int) bool {
|
||||
return messageBrokers[i].Address < messageBrokers[j].Address
|
||||
})
|
||||
|
||||
return messageBrokers
|
||||
}
|
||||
|
||||
+37
-126
@@ -5,7 +5,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -27,46 +26,9 @@ import (
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
|
||||
)
|
||||
|
||||
// FilerConfig holds filer configuration needed for bucket operations
|
||||
type FilerConfig struct {
|
||||
BucketsPath string
|
||||
FilerGroup string
|
||||
}
|
||||
|
||||
// getFilerConfig retrieves the filer configuration (buckets path and filer group)
|
||||
func (s *AdminServer) getFilerConfig() (*FilerConfig, error) {
|
||||
config := &FilerConfig{
|
||||
BucketsPath: s3_constants.DefaultBucketsPath,
|
||||
FilerGroup: "",
|
||||
}
|
||||
|
||||
err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get filer configuration: %w", err)
|
||||
}
|
||||
if resp.DirBuckets != "" {
|
||||
config.BucketsPath = resp.DirBuckets
|
||||
}
|
||||
config.FilerGroup = resp.FilerGroup
|
||||
return nil
|
||||
})
|
||||
|
||||
return config, err
|
||||
}
|
||||
|
||||
// getCollectionName returns the collection name for a bucket, prefixed with filer group if configured
|
||||
func getCollectionName(filerGroup, bucketName string) string {
|
||||
if filerGroup != "" {
|
||||
return fmt.Sprintf("%s_%s", filerGroup, bucketName)
|
||||
}
|
||||
return bucketName
|
||||
}
|
||||
|
||||
type AdminServer struct {
|
||||
masterClient *wdclient.MasterClient
|
||||
templateFS http.FileSystem
|
||||
@@ -287,17 +249,28 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
return nil, fmt.Errorf("failed to get volume information: %w", err)
|
||||
}
|
||||
|
||||
// Get filer configuration (buckets path and filer group)
|
||||
filerConfig, err := s.getFilerConfig()
|
||||
// Get filer configuration to determine FilerGroup
|
||||
var filerGroup string
|
||||
err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
configResp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to get filer configuration: %v", err)
|
||||
// Continue without filer group
|
||||
return nil
|
||||
}
|
||||
filerGroup = configResp.FilerGroup
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to get filer configuration, using defaults: %v", err)
|
||||
return nil, fmt.Errorf("failed to get filer configuration: %w", err)
|
||||
}
|
||||
|
||||
// Now list buckets from the filer and match with collection data
|
||||
err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
// List buckets by looking at the buckets directory
|
||||
// List buckets by looking at the /buckets directory
|
||||
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
|
||||
Directory: filerConfig.BucketsPath,
|
||||
Directory: "/buckets",
|
||||
Prefix: "",
|
||||
StartFromFileName: "",
|
||||
InclusiveStartFrom: false,
|
||||
@@ -320,7 +293,12 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
bucketName := resp.Entry.Name
|
||||
|
||||
// Determine collection name for this bucket
|
||||
collectionName := getCollectionName(filerConfig.FilerGroup, bucketName)
|
||||
var collectionName string
|
||||
if filerGroup != "" {
|
||||
collectionName = fmt.Sprintf("%s_%s", filerGroup, bucketName)
|
||||
} else {
|
||||
collectionName = bucketName
|
||||
}
|
||||
|
||||
// Get size and object count from collection data
|
||||
var size int64
|
||||
@@ -339,12 +317,11 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
quotaEnabled = false
|
||||
}
|
||||
|
||||
// Get versioning, object lock, and owner information from extended attributes
|
||||
// Get versioning and object lock information from extended attributes
|
||||
versioningEnabled := false
|
||||
objectLockEnabled := false
|
||||
objectLockMode := ""
|
||||
var objectLockDuration int32 = 0
|
||||
var owner string
|
||||
|
||||
if resp.Entry.Extended != nil {
|
||||
// Use shared utility to extract versioning information
|
||||
@@ -352,11 +329,6 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
|
||||
// Use shared utility to extract Object Lock information
|
||||
objectLockEnabled, objectLockMode, objectLockDuration = extractObjectLockInfoFromEntry(resp.Entry)
|
||||
|
||||
// Extract owner information
|
||||
if ownerBytes, ok := resp.Entry.Extended[s3_constants.AmzIdentityId]; ok {
|
||||
owner = string(ownerBytes)
|
||||
}
|
||||
}
|
||||
|
||||
bucket := S3Bucket{
|
||||
@@ -371,7 +343,6 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
ObjectLockEnabled: objectLockEnabled,
|
||||
ObjectLockMode: objectLockMode,
|
||||
ObjectLockDuration: objectLockDuration,
|
||||
Owner: owner,
|
||||
}
|
||||
buckets = append(buckets, bucket)
|
||||
}
|
||||
@@ -389,13 +360,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
|
||||
|
||||
// GetBucketDetails retrieves detailed information about a specific bucket
|
||||
func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error) {
|
||||
// Get filer configuration (buckets path)
|
||||
filerConfig, err := s.getFilerConfig()
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to get filer configuration, using defaults: %v", err)
|
||||
}
|
||||
|
||||
bucketPath := fmt.Sprintf("%s/%s", filerConfig.BucketsPath, bucketName)
|
||||
bucketPath := fmt.Sprintf("/buckets/%s", bucketName)
|
||||
|
||||
details := &BucketDetails{
|
||||
Bucket: S3Bucket{
|
||||
@@ -405,10 +370,10 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
|
||||
UpdatedAt: time.Now(),
|
||||
}
|
||||
|
||||
err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Get bucket info
|
||||
bucketResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: filerConfig.BucketsPath,
|
||||
Directory: "/buckets",
|
||||
Name: bucketName,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -429,12 +394,11 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
|
||||
details.Bucket.Quota = quota
|
||||
details.Bucket.QuotaEnabled = quotaEnabled
|
||||
|
||||
// Get versioning, object lock, and owner information from extended attributes
|
||||
// Get versioning and object lock information from extended attributes
|
||||
versioningEnabled := false
|
||||
objectLockEnabled := false
|
||||
objectLockMode := ""
|
||||
var objectLockDuration int32 = 0
|
||||
var owner string
|
||||
|
||||
if bucketResp.Entry.Extended != nil {
|
||||
// Use shared utility to extract versioning information
|
||||
@@ -442,21 +406,15 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
|
||||
|
||||
// Use shared utility to extract Object Lock information
|
||||
objectLockEnabled, objectLockMode, objectLockDuration = extractObjectLockInfoFromEntry(bucketResp.Entry)
|
||||
|
||||
// Extract owner information
|
||||
if ownerBytes, ok := bucketResp.Entry.Extended[s3_constants.AmzIdentityId]; ok {
|
||||
owner = string(ownerBytes)
|
||||
}
|
||||
}
|
||||
|
||||
details.Bucket.VersioningEnabled = versioningEnabled
|
||||
details.Bucket.ObjectLockEnabled = objectLockEnabled
|
||||
details.Bucket.ObjectLockMode = objectLockMode
|
||||
details.Bucket.ObjectLockDuration = objectLockDuration
|
||||
details.Bucket.Owner = owner
|
||||
|
||||
// List objects in bucket (recursively)
|
||||
return s.listBucketObjects(client, bucketPath, bucketPath, "", details)
|
||||
return s.listBucketObjects(client, bucketPath, "", details)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -467,8 +425,7 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
|
||||
}
|
||||
|
||||
// listBucketObjects recursively lists all objects in a bucket
|
||||
// bucketBasePath is the full path to the bucket (e.g., /buckets/mybucket)
|
||||
func (s *AdminServer) listBucketObjects(client filer_pb.SeaweedFilerClient, bucketBasePath, directory, prefix string, details *BucketDetails) error {
|
||||
func (s *AdminServer) listBucketObjects(client filer_pb.SeaweedFilerClient, directory, prefix string, details *BucketDetails) error {
|
||||
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
|
||||
Directory: directory,
|
||||
Prefix: prefix,
|
||||
@@ -493,16 +450,16 @@ func (s *AdminServer) listBucketObjects(client filer_pb.SeaweedFilerClient, buck
|
||||
if entry.IsDirectory {
|
||||
// Recursively list subdirectories
|
||||
subDir := fmt.Sprintf("%s/%s", directory, entry.Name)
|
||||
err := s.listBucketObjects(client, bucketBasePath, subDir, "", details)
|
||||
err := s.listBucketObjects(client, subDir, "", details)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// Add file object
|
||||
objectKey := entry.Name
|
||||
if directory != bucketBasePath {
|
||||
if directory != fmt.Sprintf("/buckets/%s", details.Bucket.Name) {
|
||||
// Remove bucket prefix to get relative path
|
||||
relativePath := directory[len(bucketBasePath)+1:]
|
||||
relativePath := directory[len(fmt.Sprintf("/buckets/%s", details.Bucket.Name))+1:]
|
||||
objectKey = fmt.Sprintf("%s/%s", relativePath, entry.Name)
|
||||
}
|
||||
|
||||
@@ -534,45 +491,14 @@ func (s *AdminServer) CreateS3Bucket(bucketName string) error {
|
||||
|
||||
// DeleteS3Bucket deletes an S3 bucket and all its contents
|
||||
func (s *AdminServer) DeleteS3Bucket(bucketName string) error {
|
||||
ctx := context.Background()
|
||||
|
||||
// Get filer configuration (buckets path and filer group)
|
||||
filerConfig, err := s.getFilerConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get filer configuration: %w", err)
|
||||
}
|
||||
|
||||
// Check if bucket has Object Lock enabled and if there are locked objects
|
||||
err = s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
return s3api.CheckBucketForLockedObjects(ctx, client, filerConfig.BucketsPath, bucketName)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete the collection first (same as s3.bucket.delete shell command)
|
||||
// This ensures volume data is cleaned up properly
|
||||
// Collection name must be prefixed with filer group if configured
|
||||
collectionName := getCollectionName(filerConfig.FilerGroup, bucketName)
|
||||
err = s.WithMasterClient(func(client master_pb.SeaweedClient) error {
|
||||
_, err := client.CollectionDelete(ctx, &master_pb.CollectionDeleteRequest{
|
||||
Name: collectionName,
|
||||
})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete collection %s: %w", collectionName, err)
|
||||
}
|
||||
|
||||
// Then delete bucket directory recursively from filer
|
||||
// Use same parameters as s3.bucket.delete shell command and S3 API
|
||||
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
_, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
|
||||
Directory: filerConfig.BucketsPath,
|
||||
// Delete bucket directory recursively
|
||||
_, err := client.DeleteEntry(context.Background(), &filer_pb.DeleteEntryRequest{
|
||||
Directory: "/buckets",
|
||||
Name: bucketName,
|
||||
IsDeleteData: false, // Collection already deleted, just remove metadata
|
||||
IsDeleteData: true,
|
||||
IsRecursive: true,
|
||||
IgnoreRecursiveError: true, // Same as S3 API and shell command
|
||||
IgnoreRecursiveError: false,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to delete bucket: %w", err)
|
||||
@@ -721,11 +647,6 @@ func (s *AdminServer) GetClusterMasters() (*ClusterMastersData, error) {
|
||||
masters = append(masters, *masterInfo)
|
||||
}
|
||||
|
||||
// Sort masters by address for consistent ordering on page refresh
|
||||
sort.Slice(masters, func(i, j int) bool {
|
||||
return masters[i].Address < masters[j].Address
|
||||
})
|
||||
|
||||
// If no masters found at all, add the current master as fallback
|
||||
if len(masters) == 0 {
|
||||
currentMaster := s.masterClient.GetMaster(context.Background())
|
||||
@@ -782,11 +703,6 @@ func (s *AdminServer) GetClusterFilers() (*ClusterFilersData, error) {
|
||||
return nil, fmt.Errorf("failed to get filer nodes from master: %w", err)
|
||||
}
|
||||
|
||||
// Sort filers by address for consistent ordering on page refresh
|
||||
sort.Slice(filers, func(i, j int) bool {
|
||||
return filers[i].Address < filers[j].Address
|
||||
})
|
||||
|
||||
return &ClusterFilersData{
|
||||
Filers: filers,
|
||||
TotalFilers: len(filers),
|
||||
@@ -829,11 +745,6 @@ func (s *AdminServer) GetClusterBrokers() (*ClusterBrokersData, error) {
|
||||
return nil, fmt.Errorf("failed to get broker nodes from master: %w", err)
|
||||
}
|
||||
|
||||
// Sort brokers by address for consistent ordering on page refresh
|
||||
sort.Slice(brokers, func(i, j int) bool {
|
||||
return brokers[i].Address < brokers[j].Address
|
||||
})
|
||||
|
||||
return &ClusterBrokersData{
|
||||
Brokers: brokers,
|
||||
TotalBrokers: len(brokers),
|
||||
|
||||
@@ -11,14 +11,8 @@ import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
)
|
||||
|
||||
// MaxOwnerNameLength is the maximum allowed length for bucket owner identity names.
|
||||
// This is a reasonable limit to prevent abuse; AWS IAM user names are limited to 64 chars,
|
||||
// but we use 256 to allow for more complex identity formats (e.g., email addresses).
|
||||
const MaxOwnerNameLength = 256
|
||||
|
||||
// S3 Bucket management data structures for templates
|
||||
type S3BucketsData struct {
|
||||
Username string `json:"username"`
|
||||
@@ -39,7 +33,6 @@ type CreateBucketRequest struct {
|
||||
ObjectLockMode string `json:"object_lock_mode"` // Object lock mode: "GOVERNANCE" or "COMPLIANCE"
|
||||
SetDefaultRetention bool `json:"set_default_retention"` // Whether to set default retention
|
||||
ObjectLockDuration int32 `json:"object_lock_duration"` // Default retention duration in days
|
||||
Owner string `json:"owner"` // Bucket owner identity (for S3 IAM authentication)
|
||||
}
|
||||
|
||||
// S3 Bucket Management Handlers
|
||||
@@ -125,14 +118,7 @@ func (s *AdminServer) CreateBucket(c *gin.Context) {
|
||||
// Convert quota to bytes
|
||||
quotaBytes := convertQuotaToBytes(req.QuotaSize, req.QuotaUnit)
|
||||
|
||||
// Sanitize owner: trim whitespace and enforce max length
|
||||
owner := strings.TrimSpace(req.Owner)
|
||||
if len(owner) > MaxOwnerNameLength {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength)})
|
||||
return
|
||||
}
|
||||
|
||||
err := s.CreateS3BucketWithObjectLock(req.Name, quotaBytes, req.QuotaEnabled, req.VersioningEnabled, req.ObjectLockEnabled, req.ObjectLockMode, req.SetDefaultRetention, req.ObjectLockDuration, owner)
|
||||
err := s.CreateS3BucketWithObjectLock(req.Name, quotaBytes, req.QuotaEnabled, req.VersioningEnabled, req.ObjectLockEnabled, req.ObjectLockMode, req.SetDefaultRetention, req.ObjectLockDuration)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create bucket: " + err.Error()})
|
||||
return
|
||||
@@ -148,7 +134,6 @@ func (s *AdminServer) CreateBucket(c *gin.Context) {
|
||||
"object_lock_enabled": req.ObjectLockEnabled,
|
||||
"object_lock_mode": req.ObjectLockMode,
|
||||
"object_lock_duration": req.ObjectLockDuration,
|
||||
"owner": owner,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -208,88 +193,6 @@ func (s *AdminServer) DeleteBucket(c *gin.Context) {
|
||||
})
|
||||
}
|
||||
|
||||
// UpdateBucketOwner updates the owner of an S3 bucket
|
||||
func (s *AdminServer) UpdateBucketOwner(c *gin.Context) {
|
||||
bucketName := c.Param("bucket")
|
||||
if bucketName == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Bucket name is required"})
|
||||
return
|
||||
}
|
||||
|
||||
// Use pointer to detect if owner field was explicitly provided
|
||||
var req struct {
|
||||
Owner *string `json:"owner"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Require owner field to be explicitly provided
|
||||
if req.Owner == nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "Owner field is required (use empty string to clear owner)"})
|
||||
return
|
||||
}
|
||||
|
||||
// Trim and validate owner
|
||||
owner := strings.TrimSpace(*req.Owner)
|
||||
if len(owner) > MaxOwnerNameLength {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Owner name must be %d characters or less", MaxOwnerNameLength)})
|
||||
return
|
||||
}
|
||||
|
||||
err := s.SetBucketOwner(bucketName, owner)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update bucket owner: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"message": "Bucket owner updated successfully",
|
||||
"bucket": bucketName,
|
||||
"owner": owner,
|
||||
})
|
||||
}
|
||||
|
||||
// SetBucketOwner sets the owner of a bucket
|
||||
func (s *AdminServer) SetBucketOwner(bucketName string, owner string) error {
|
||||
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
// Get the current bucket entry
|
||||
lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
|
||||
Directory: "/buckets",
|
||||
Name: bucketName,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("lookup bucket %s: %w", bucketName, err)
|
||||
}
|
||||
|
||||
bucketEntry := lookupResp.Entry
|
||||
|
||||
// Initialize Extended map if nil
|
||||
if bucketEntry.Extended == nil {
|
||||
bucketEntry.Extended = make(map[string][]byte)
|
||||
}
|
||||
|
||||
// Set or remove the owner
|
||||
if owner == "" {
|
||||
delete(bucketEntry.Extended, s3_constants.AmzIdentityId)
|
||||
} else {
|
||||
bucketEntry.Extended[s3_constants.AmzIdentityId] = []byte(owner)
|
||||
}
|
||||
|
||||
// Update the entry
|
||||
_, err = client.UpdateEntry(context.Background(), &filer_pb.UpdateEntryRequest{
|
||||
Directory: "/buckets",
|
||||
Entry: bucketEntry,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update bucket owner: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// ListBucketsAPI returns the list of buckets as JSON
|
||||
func (s *AdminServer) ListBucketsAPI(c *gin.Context) {
|
||||
buckets, err := s.GetS3Buckets()
|
||||
@@ -385,11 +288,11 @@ func (s *AdminServer) SetBucketQuota(bucketName string, quotaBytes int64, quotaE
|
||||
|
||||
// CreateS3BucketWithQuota creates a new S3 bucket with quota settings
|
||||
func (s *AdminServer) CreateS3BucketWithQuota(bucketName string, quotaBytes int64, quotaEnabled bool) error {
|
||||
return s.CreateS3BucketWithObjectLock(bucketName, quotaBytes, quotaEnabled, false, false, "", false, 0, "")
|
||||
return s.CreateS3BucketWithObjectLock(bucketName, quotaBytes, quotaEnabled, false, false, "", false, 0)
|
||||
}
|
||||
|
||||
// CreateS3BucketWithObjectLock creates a new S3 bucket with quota, versioning, object lock settings, and owner
|
||||
func (s *AdminServer) CreateS3BucketWithObjectLock(bucketName string, quotaBytes int64, quotaEnabled, versioningEnabled, objectLockEnabled bool, objectLockMode string, setDefaultRetention bool, objectLockDuration int32, owner string) error {
|
||||
// CreateS3BucketWithObjectLock creates a new S3 bucket with quota, versioning, and object lock settings
|
||||
func (s *AdminServer) CreateS3BucketWithObjectLock(bucketName string, quotaBytes int64, quotaEnabled, versioningEnabled, objectLockEnabled bool, objectLockMode string, setDefaultRetention bool, objectLockDuration int32) error {
|
||||
return s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
|
||||
// First ensure /buckets directory exists
|
||||
_, err := client.CreateEntry(context.Background(), &filer_pb.CreateEntryRequest{
|
||||
@@ -441,14 +344,9 @@ func (s *AdminServer) CreateS3BucketWithObjectLock(bucketName string, quotaBytes
|
||||
TtlSec: 0,
|
||||
}
|
||||
|
||||
// Create extended attributes map for versioning and owner
|
||||
// Create extended attributes map for versioning
|
||||
extended := make(map[string][]byte)
|
||||
|
||||
// Set bucket owner if specified
|
||||
if owner != "" {
|
||||
extended[s3_constants.AmzIdentityId] = []byte(owner)
|
||||
}
|
||||
|
||||
// Create bucket entry
|
||||
bucketEntry := &filer_pb.Entry{
|
||||
Name: bucketName,
|
||||
|
||||
@@ -82,7 +82,6 @@ type S3Bucket struct {
|
||||
ObjectLockEnabled bool `json:"object_lock_enabled"` // Whether object lock is enabled
|
||||
ObjectLockMode string `json:"object_lock_mode"` // Object lock mode: "GOVERNANCE" or "COMPLIANCE"
|
||||
ObjectLockDuration int32 `json:"object_lock_duration"` // Default retention duration in days
|
||||
Owner string `json:"owner,omitempty"` // Bucket owner identity; empty means admin-only access
|
||||
}
|
||||
|
||||
type S3Object struct {
|
||||
|
||||
@@ -544,11 +544,6 @@ func (s *AdminServer) GetClusterVolumeServers() (*ClusterVolumeServersData, erro
|
||||
volumeServers = append(volumeServers, *vs)
|
||||
}
|
||||
|
||||
// Sort volume servers by address for consistent ordering on page refresh
|
||||
sort.Slice(volumeServers, func(i, j int) bool {
|
||||
return volumeServers[i].GetDisplayAddress() < volumeServers[j].GetDisplayAddress()
|
||||
})
|
||||
|
||||
var totalCapacity int64
|
||||
var totalVolumes int
|
||||
for _, vs := range volumeServers {
|
||||
|
||||
@@ -117,7 +117,7 @@ func (s *WorkerGrpcServer) Stop() error {
|
||||
s.connMutex.Lock()
|
||||
for _, conn := range s.connections {
|
||||
conn.cancel()
|
||||
s.safeCloseOutgoingChannel(conn, "Stop")
|
||||
close(conn.outgoing)
|
||||
}
|
||||
s.connections = make(map[string]*WorkerConnection)
|
||||
s.connMutex.Unlock()
|
||||
@@ -182,27 +182,15 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
|
||||
}
|
||||
conn.capabilities = capabilities
|
||||
|
||||
// Register connection - clean up old connection if worker is reconnecting
|
||||
// Register connection
|
||||
s.connMutex.Lock()
|
||||
if oldConn, exists := s.connections[workerID]; exists {
|
||||
glog.Infof("Worker %s reconnected, cleaning up old connection", workerID)
|
||||
// Cancel old connection to stop its goroutines
|
||||
oldConn.cancel()
|
||||
// Don't close oldConn.outgoing here as it may cause panic in handleOutgoingMessages
|
||||
// Let the goroutine exit naturally when it detects context cancellation
|
||||
}
|
||||
s.connections[workerID] = conn
|
||||
s.connMutex.Unlock()
|
||||
|
||||
// Register worker with maintenance manager
|
||||
s.registerWorkerWithManager(conn)
|
||||
|
||||
// IMPORTANT: Start outgoing message handler BEFORE sending registration response
|
||||
// This ensures the handler is ready to process messages and prevents race conditions
|
||||
// where the worker might send requests before we're ready to respond
|
||||
go s.handleOutgoingMessages(conn)
|
||||
|
||||
// Send registration response (after handler is started)
|
||||
// Send registration response
|
||||
regResponse := &worker_pb.AdminMessage{
|
||||
Timestamp: time.Now().Unix(),
|
||||
Message: &worker_pb.AdminMessage_RegistrationResponse{
|
||||
@@ -215,11 +203,13 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
|
||||
|
||||
select {
|
||||
case conn.outgoing <- regResponse:
|
||||
glog.V(1).Infof("Registration response sent to worker %s", workerID)
|
||||
case <-time.After(5 * time.Second):
|
||||
glog.Errorf("Failed to send registration response to worker %s", workerID)
|
||||
}
|
||||
|
||||
// Start outgoing message handler
|
||||
go s.handleOutgoingMessages(conn)
|
||||
|
||||
// Handle incoming messages
|
||||
for {
|
||||
select {
|
||||
@@ -245,9 +235,7 @@ func (s *WorkerGrpcServer) WorkerStream(stream worker_pb.WorkerService_WorkerStr
|
||||
return err
|
||||
}
|
||||
|
||||
s.connMutex.Lock()
|
||||
conn.lastSeen = time.Now()
|
||||
s.connMutex.Unlock()
|
||||
s.handleWorkerMessage(conn, msg)
|
||||
}
|
||||
}
|
||||
@@ -452,36 +440,16 @@ func (s *WorkerGrpcServer) handleTaskLogResponse(conn *WorkerConnection, respons
|
||||
s.logRequestsMutex.Unlock()
|
||||
}
|
||||
|
||||
// safeCloseOutgoingChannel safely closes the outgoing channel for a worker connection.
|
||||
func (s *WorkerGrpcServer) safeCloseOutgoingChannel(conn *WorkerConnection, source string) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
glog.V(1).Infof("%s: recovered from panic closing outgoing channel for worker %s: %v", source, conn.workerID, r)
|
||||
}
|
||||
}()
|
||||
close(conn.outgoing)
|
||||
}
|
||||
|
||||
// unregisterWorker removes a worker connection
|
||||
func (s *WorkerGrpcServer) unregisterWorker(workerID string) {
|
||||
s.connMutex.Lock()
|
||||
conn, exists := s.connections[workerID]
|
||||
if !exists {
|
||||
s.connMutex.Unlock()
|
||||
glog.V(2).Infof("unregisterWorker: worker %s not found in connections map (already unregistered)", workerID)
|
||||
return
|
||||
if conn, exists := s.connections[workerID]; exists {
|
||||
conn.cancel()
|
||||
close(conn.outgoing)
|
||||
delete(s.connections, workerID)
|
||||
}
|
||||
|
||||
// Remove from map first to prevent duplicate cleanup attempts
|
||||
delete(s.connections, workerID)
|
||||
s.connMutex.Unlock()
|
||||
|
||||
// Cancel context to signal goroutines to stop
|
||||
conn.cancel()
|
||||
|
||||
// Safely close the outgoing channel with recover to handle potential double-close
|
||||
s.safeCloseOutgoingChannel(conn, "unregisterWorker")
|
||||
|
||||
glog.V(1).Infof("Unregistered worker %s", workerID)
|
||||
}
|
||||
|
||||
@@ -511,7 +479,7 @@ func (s *WorkerGrpcServer) cleanupStaleConnections() {
|
||||
if conn.lastSeen.Before(cutoff) {
|
||||
glog.Warningf("Cleaning up stale worker connection: %s", workerID)
|
||||
conn.cancel()
|
||||
s.safeCloseOutgoingChannel(conn, "cleanupStaleConnections")
|
||||
close(conn.outgoing)
|
||||
delete(s.connections, workerID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +119,6 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
|
||||
s3Api.DELETE("/buckets/:bucket", h.adminServer.DeleteBucket)
|
||||
s3Api.GET("/buckets/:bucket", h.adminServer.ShowBucketDetails)
|
||||
s3Api.PUT("/buckets/:bucket/quota", h.adminServer.UpdateBucketQuota)
|
||||
s3Api.PUT("/buckets/:bucket/owner", h.adminServer.UpdateBucketOwner)
|
||||
}
|
||||
|
||||
// User management API routes
|
||||
@@ -246,7 +245,6 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
|
||||
s3Api.DELETE("/buckets/:bucket", h.adminServer.DeleteBucket)
|
||||
s3Api.GET("/buckets/:bucket", h.adminServer.ShowBucketDetails)
|
||||
s3Api.PUT("/buckets/:bucket/quota", h.adminServer.UpdateBucketQuota)
|
||||
s3Api.PUT("/buckets/:bucket/owner", h.adminServer.UpdateBucketOwner)
|
||||
}
|
||||
|
||||
// User management API routes
|
||||
|
||||
@@ -266,26 +266,7 @@ func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.Vo
|
||||
|
||||
// UpdateTopologyInfo updates the volume shard tracker with topology information for empty servers
|
||||
func (s *MaintenanceIntegration) UpdateTopologyInfo(topologyInfo *master_pb.TopologyInfo) error {
|
||||
// Log topology details before update for diagnostics
|
||||
if topologyInfo != nil {
|
||||
dcCount, nodeCount, diskCount := topology.CountTopologyResources(topologyInfo)
|
||||
glog.V(2).Infof("UpdateTopologyInfo: received topology with %d datacenters, %d nodes, %d disks",
|
||||
dcCount, nodeCount, diskCount)
|
||||
} else {
|
||||
glog.Warningf("UpdateTopologyInfo: received nil topologyInfo")
|
||||
}
|
||||
|
||||
err := s.activeTopology.UpdateTopology(topologyInfo)
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("UpdateTopologyInfo: topology update failed: %v", err)
|
||||
} else {
|
||||
// Log success with current disk count
|
||||
currentDiskCount := s.activeTopology.GetDiskCount()
|
||||
glog.V(1).Infof("UpdateTopologyInfo: topology update successful, active topology now has %d disks", currentDiskCount)
|
||||
}
|
||||
|
||||
return err
|
||||
return s.activeTopology.UpdateTopology(topologyInfo)
|
||||
}
|
||||
|
||||
// convertToExistingFormat converts task results to existing system format using dynamic mapping
|
||||
|
||||
@@ -137,7 +137,6 @@ func (mm *MaintenanceManager) Start() error {
|
||||
// Start background processes
|
||||
go mm.scanLoop()
|
||||
go mm.cleanupLoop()
|
||||
go mm.topologyStatusLoop() // Periodic diagnostic logging
|
||||
|
||||
glog.Infof("Maintenance manager started with scan interval %ds", mm.config.ScanIntervalSeconds)
|
||||
return nil
|
||||
@@ -256,54 +255,6 @@ func (mm *MaintenanceManager) cleanupLoop() {
|
||||
}
|
||||
}
|
||||
|
||||
// topologyStatusLoop periodically logs topology status for diagnostics
|
||||
func (mm *MaintenanceManager) topologyStatusLoop() {
|
||||
// Log topology status every 5 minutes for diagnostic purposes
|
||||
statusInterval := 5 * time.Minute
|
||||
ticker := time.NewTicker(statusInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for mm.running {
|
||||
select {
|
||||
case <-mm.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
mm.logTopologyStatus()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// logTopologyStatus logs current topology and worker status for diagnostics
|
||||
func (mm *MaintenanceManager) logTopologyStatus() {
|
||||
if mm.scanner == nil || mm.scanner.integration == nil {
|
||||
glog.V(2).Infof("Topology status: scanner/integration not available")
|
||||
return
|
||||
}
|
||||
|
||||
activeTopology := mm.scanner.integration.GetActiveTopology()
|
||||
if activeTopology == nil {
|
||||
glog.V(1).Infof("Topology status: ActiveTopology is nil")
|
||||
return
|
||||
}
|
||||
|
||||
diskCount := activeTopology.GetDiskCount()
|
||||
nodeCount := len(activeTopology.GetAllNodes())
|
||||
|
||||
// Get queue stats
|
||||
stats := mm.queue.GetStats()
|
||||
workerCount := len(mm.queue.GetWorkers())
|
||||
|
||||
mm.mutex.RLock()
|
||||
errorCount := mm.errorCount
|
||||
mm.mutex.RUnlock()
|
||||
|
||||
glog.V(0).Infof("Topology status: %d nodes, %d disks, %d workers, %d pending tasks, %d running tasks, errors: %d",
|
||||
nodeCount, diskCount, workerCount,
|
||||
stats.TasksByStatus[TaskStatusPending],
|
||||
stats.TasksByStatus[TaskStatusInProgress]+stats.TasksByStatus[TaskStatusAssigned],
|
||||
errorCount)
|
||||
}
|
||||
|
||||
// performScan executes a maintenance scan with error handling and backoff
|
||||
func (mm *MaintenanceManager) performScan() {
|
||||
defer func() {
|
||||
|
||||
@@ -8,53 +8,11 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
)
|
||||
|
||||
// CountTopologyResources counts datacenters, nodes, and disks in topology info
|
||||
func CountTopologyResources(topologyInfo *master_pb.TopologyInfo) (dcCount, nodeCount, diskCount int) {
|
||||
if topologyInfo == nil {
|
||||
return 0, 0, 0
|
||||
}
|
||||
dcCount = len(topologyInfo.DataCenterInfos)
|
||||
for _, dc := range topologyInfo.DataCenterInfos {
|
||||
for _, rack := range dc.RackInfos {
|
||||
nodeCount += len(rack.DataNodeInfos)
|
||||
for _, node := range rack.DataNodeInfos {
|
||||
diskCount += len(node.DiskInfos)
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UpdateTopology updates the topology information from master
|
||||
func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) error {
|
||||
at.mutex.Lock()
|
||||
defer at.mutex.Unlock()
|
||||
|
||||
// Validate topology updates to prevent clearing disk maps with invalid data
|
||||
if topologyInfo == nil {
|
||||
glog.Warningf("UpdateTopology received nil topologyInfo, preserving last-known-good topology")
|
||||
return fmt.Errorf("rejected invalid topology update: nil topologyInfo")
|
||||
}
|
||||
|
||||
if len(topologyInfo.DataCenterInfos) == 0 {
|
||||
glog.Warningf("UpdateTopology received empty DataCenterInfos, preserving last-known-good topology (had %d nodes, %d disks)",
|
||||
len(at.nodes), len(at.disks))
|
||||
return fmt.Errorf("rejected invalid topology update: empty DataCenterInfos (had %d nodes, %d disks)", len(at.nodes), len(at.disks))
|
||||
}
|
||||
|
||||
// Count incoming topology for validation logging
|
||||
dcCount, incomingNodes, incomingDisks := CountTopologyResources(topologyInfo)
|
||||
|
||||
// Reject updates that would wipe out a valid topology with an empty one (e.g. during master restart)
|
||||
if incomingNodes == 0 && len(at.nodes) > 0 {
|
||||
glog.Warningf("UpdateTopology received topology with 0 nodes, preserving last-known-good topology (had %d nodes, %d disks)",
|
||||
len(at.nodes), len(at.disks))
|
||||
return fmt.Errorf("rejected invalid topology update: 0 nodes (had %d nodes, %d disks)", len(at.nodes), len(at.disks))
|
||||
}
|
||||
|
||||
glog.V(2).Infof("UpdateTopology: validating update with %d datacenters, %d nodes, %d disks (current: %d nodes, %d disks)",
|
||||
dcCount, incomingNodes, incomingDisks, len(at.nodes), len(at.disks))
|
||||
|
||||
at.topologyInfo = topologyInfo
|
||||
at.lastUpdated = time.Now()
|
||||
|
||||
@@ -184,21 +142,8 @@ func (at *ActiveTopology) GetNodeDisks(nodeID string) []*DiskInfo {
|
||||
return disks
|
||||
}
|
||||
|
||||
// GetDiskCount returns the total number of disks in the active topology
|
||||
func (at *ActiveTopology) GetDiskCount() int {
|
||||
at.mutex.RLock()
|
||||
defer at.mutex.RUnlock()
|
||||
return len(at.disks)
|
||||
}
|
||||
|
||||
// rebuildIndexes rebuilds the volume and EC shard indexes for O(1) lookups
|
||||
func (at *ActiveTopology) rebuildIndexes() {
|
||||
// Nil-safety guard: return early if topology is not valid
|
||||
if at.topologyInfo == nil || at.topologyInfo.DataCenterInfos == nil {
|
||||
glog.V(1).Infof("rebuildIndexes: skipping rebuild due to nil topology or DataCenterInfos")
|
||||
return
|
||||
}
|
||||
|
||||
// Clear existing indexes
|
||||
at.volumeIndex = make(map[uint32][]string)
|
||||
at.ecShardIndex = make(map[uint32][]string)
|
||||
|
||||
@@ -2,149 +2,154 @@ package app
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
)
|
||||
|
||||
templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
|
||||
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
|
||||
<div>
|
||||
<h1 class="h2">
|
||||
<i class="fas fa-database me-2"></i>EC Volumes
|
||||
</h1>
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<title>EC Volumes - SeaweedFS</title>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
|
||||
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css" rel="stylesheet">
|
||||
</head>
|
||||
<body>
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<div class="col-12">
|
||||
<h2 class="mb-4">
|
||||
<i class="fas fa-database me-2"></i>EC Volumes
|
||||
<small class="text-muted">({fmt.Sprintf("%d", data.TotalVolumes)} volumes)</small>
|
||||
</h2>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Statistics Cards -->
|
||||
<div class="row mb-4">
|
||||
<div class="col-md-3">
|
||||
<div class="card text-bg-primary">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between">
|
||||
<div>
|
||||
<h6 class="card-title">Total Volumes</h6>
|
||||
<h4 class="mb-0">{fmt.Sprintf("%d", data.TotalVolumes)}</h4>
|
||||
<small>EC encoded volumes</small>
|
||||
</div>
|
||||
<div class="align-self-center">
|
||||
<i class="fas fa-cubes fa-2x"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<div class="card text-bg-info">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between">
|
||||
<div>
|
||||
<h6 class="card-title">Total Shards</h6>
|
||||
<h4 class="mb-0">{fmt.Sprintf("%d", data.TotalShards)}</h4>
|
||||
<small>Distributed shards</small>
|
||||
</div>
|
||||
<div class="align-self-center">
|
||||
<i class="fas fa-puzzle-piece fa-2x"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<div class="card text-bg-success">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between">
|
||||
<div>
|
||||
<h6 class="card-title">Complete Volumes</h6>
|
||||
<h4 class="mb-0">{fmt.Sprintf("%d", data.CompleteVolumes)}</h4>
|
||||
<small>All shards present</small>
|
||||
</div>
|
||||
<div class="align-self-center">
|
||||
<i class="fas fa-check-circle fa-2x"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<div class="card text-bg-warning">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between">
|
||||
<div>
|
||||
<h6 class="card-title">Incomplete Volumes</h6>
|
||||
<h4 class="mb-0">{fmt.Sprintf("%d", data.IncompleteVolumes)}</h4>
|
||||
<small>Missing shards</small>
|
||||
</div>
|
||||
<div class="align-self-center">
|
||||
<i class="fas fa-exclamation-triangle fa-2x"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- EC Storage Information Note -->
|
||||
<div class="alert alert-info mb-4" role="alert">
|
||||
<i class="fas fa-info-circle me-2"></i>
|
||||
<strong>EC Storage Note:</strong>
|
||||
EC volumes use erasure coding ({ fmt.Sprintf("%d+%d", erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount) }) which stores data across { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards with redundancy.
|
||||
Physical storage is approximately { fmt.Sprintf("%.1fx", float64(erasure_coding.TotalShardsCount)/float64(erasure_coding.DataShardsCount)) } the original logical data size due to { fmt.Sprintf("%d", erasure_coding.ParityShardsCount) } parity shards.
|
||||
</div>
|
||||
|
||||
<!-- Volumes Table -->
|
||||
<div class="d-flex justify-content-between align-items-center mb-3">
|
||||
<div class="d-flex align-items-center">
|
||||
<span class="me-3">
|
||||
Showing {fmt.Sprintf("%d", (data.Page-1)*data.PageSize + 1)} to {fmt.Sprintf("%d", func() int {
|
||||
end := data.Page * data.PageSize
|
||||
if end > data.TotalVolumes {
|
||||
return data.TotalVolumes
|
||||
}
|
||||
return end
|
||||
}())} of {fmt.Sprintf("%d", data.TotalVolumes)} volumes
|
||||
</span>
|
||||
|
||||
<div class="d-flex align-items-center">
|
||||
<label for="pageSize" class="form-label me-2 mb-0">Show:</label>
|
||||
<select id="pageSize" class="form-select form-select-sm" style="width: auto;" onchange="changePageSize(this.value)">
|
||||
<option value="5" if data.PageSize == 5 { selected }>5</option>
|
||||
<option value="10" if data.PageSize == 10 { selected }>10</option>
|
||||
<option value="25" if data.PageSize == 25 { selected }>25</option>
|
||||
<option value="50" if data.PageSize == 50 { selected }>50</option>
|
||||
<option value="100" if data.PageSize == 100 { selected }>100</option>
|
||||
</select>
|
||||
<span class="ms-2">per page</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
if data.Collection != "" {
|
||||
<div class="d-flex align-items-center mt-2">
|
||||
<div>
|
||||
if data.Collection == "default" {
|
||||
<span class="badge bg-secondary text-white me-2">
|
||||
<i class="fas fa-filter me-1"></i>Collection: default
|
||||
</span>
|
||||
<span class="badge bg-secondary text-white">Collection: default</span>
|
||||
} else {
|
||||
<span class="badge bg-info text-white me-2">
|
||||
<i class="fas fa-filter me-1"></i>Collection: {data.Collection}
|
||||
</span>
|
||||
<span class="badge bg-info text-white">Collection: {data.Collection}</span>
|
||||
}
|
||||
<a href="/cluster/ec-volumes" class="btn btn-sm btn-outline-secondary">
|
||||
<i class="fas fa-times me-1"></i>Clear Filter
|
||||
</a>
|
||||
<a href="/cluster/ec-shards" class="btn btn-sm btn-outline-secondary ms-2">Clear Filter</a>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
<div class="btn-toolbar mb-2 mb-md-0">
|
||||
<div class="btn-group me-2">
|
||||
<select class="form-select form-select-sm me-2" id="pageSizeSelect" onchange="changePageSize(this.value)" style="width: auto;">
|
||||
<option value="5" if data.PageSize == 5 { selected="selected" }>5 per page</option>
|
||||
<option value="10" if data.PageSize == 10 { selected="selected" }>10 per page</option>
|
||||
<option value="25" if data.PageSize == 25 { selected="selected" }>25 per page</option>
|
||||
<option value="50" if data.PageSize == 50 { selected="selected" }>50 per page</option>
|
||||
<option value="100" if data.PageSize == 100 { selected="selected" }>100 per page</option>
|
||||
</select>
|
||||
<button type="button" class="btn btn-sm btn-outline-primary" onclick="window.location.reload()">
|
||||
<i class="fas fa-refresh me-1"></i>Refresh
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Statistics Cards -->
|
||||
<div class="row mb-4">
|
||||
<div class="col-md-3">
|
||||
<div class="card text-bg-primary">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between">
|
||||
<div>
|
||||
<h6 class="card-title">Total Volumes</h6>
|
||||
<h4 class="mb-0">{fmt.Sprintf("%d", data.TotalVolumes)}</h4>
|
||||
</div>
|
||||
<div class="align-self-center">
|
||||
<i class="fas fa-cubes fa-2x"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<div class="card text-bg-info">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between">
|
||||
<div>
|
||||
<h6 class="card-title">Total Shards</h6>
|
||||
<h4 class="mb-0">{fmt.Sprintf("%d", data.TotalShards)}</h4>
|
||||
</div>
|
||||
<div class="align-self-center">
|
||||
<i class="fas fa-puzzle-piece fa-2x"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<div class="card text-bg-success">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between">
|
||||
<div>
|
||||
<h6 class="card-title">Healthy Volumes</h6>
|
||||
<h4 class="mb-0">{fmt.Sprintf("%d", data.CompleteVolumes)}</h4>
|
||||
<small>All { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards present</small>
|
||||
</div>
|
||||
<div class="align-self-center">
|
||||
<i class="fas fa-check-circle fa-2x"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-3">
|
||||
<div class="card text-bg-warning">
|
||||
<div class="card-body">
|
||||
<div class="d-flex justify-content-between">
|
||||
<div>
|
||||
<h6 class="card-title">Degraded Volumes</h6>
|
||||
<h4 class="mb-0">{fmt.Sprintf("%d", data.IncompleteVolumes)}</h4>
|
||||
<small>Incomplete/Critical</small>
|
||||
</div>
|
||||
<div class="align-self-center">
|
||||
<i class="fas fa-exclamation-triangle fa-2x"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- EC Storage Information Note -->
|
||||
<div class="alert alert-info mb-4" role="alert">
|
||||
<i class="fas fa-info-circle me-2"></i>
|
||||
<strong>EC Storage Note:</strong>
|
||||
EC volumes use erasure coding ({ fmt.Sprintf("%d+%d", erasure_coding.DataShardsCount, erasure_coding.ParityShardsCount) }) which stores data across { fmt.Sprintf("%d", erasure_coding.TotalShardsCount) } shards with redundancy.
|
||||
Physical storage is approximately { fmt.Sprintf("%.1fx", float64(erasure_coding.TotalShardsCount)/float64(erasure_coding.DataShardsCount)) } the original logical data size due to { fmt.Sprintf("%d", erasure_coding.ParityShardsCount) } parity shards.
|
||||
</div>
|
||||
|
||||
<!-- Volumes Table -->
|
||||
<div class="table-responsive">
|
||||
<table class="table table-striped table-hover" id="ecVolumesTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>
|
||||
<a href="#" onclick="sortBy('volume_id')" class="text-dark text-decoration-none">
|
||||
Volume ID
|
||||
if data.SortBy == "volume_id" {
|
||||
if data.SortOrder == "asc" {
|
||||
<i class="fas fa-sort-up ms-1"></i>
|
||||
} else {
|
||||
<i class="fas fa-sort-down ms-1"></i>
|
||||
}
|
||||
} else {
|
||||
<i class="fas fa-sort ms-1 text-muted"></i>
|
||||
}
|
||||
</a>
|
||||
</th>
|
||||
if data.ShowCollectionColumn {
|
||||
<div class="table-responsive">
|
||||
<table class="table table-striped table-hover" id="ecVolumesTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>
|
||||
<a href="#" onclick="sortBy('collection')" class="text-dark text-decoration-none">
|
||||
Collection
|
||||
if data.SortBy == "collection" {
|
||||
<a href="#" onclick="sortBy('volume_id')" class="text-dark text-decoration-none">
|
||||
Volume ID
|
||||
if data.SortBy == "volume_id" {
|
||||
if data.SortOrder == "asc" {
|
||||
<i class="fas fa-sort-up ms-1"></i>
|
||||
} else {
|
||||
@@ -155,184 +160,172 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
|
||||
}
|
||||
</a>
|
||||
</th>
|
||||
}
|
||||
<th>
|
||||
<a href="#" onclick="sortBy('total_shards')" class="text-dark text-decoration-none">
|
||||
Shard Count
|
||||
if data.SortBy == "total_shards" {
|
||||
if data.SortOrder == "asc" {
|
||||
<i class="fas fa-sort-up ms-1"></i>
|
||||
} else {
|
||||
<i class="fas fa-sort-down ms-1"></i>
|
||||
}
|
||||
} else {
|
||||
<i class="fas fa-sort ms-1 text-muted"></i>
|
||||
}
|
||||
</a>
|
||||
</th>
|
||||
<th class="text-dark">Shard Size</th>
|
||||
<th class="text-dark">Shard Locations</th>
|
||||
<th>
|
||||
<a href="#" onclick="sortBy('completeness')" class="text-dark text-decoration-none">
|
||||
Status
|
||||
if data.SortBy == "completeness" {
|
||||
if data.SortOrder == "asc" {
|
||||
<i class="fas fa-sort-up ms-1"></i>
|
||||
} else {
|
||||
<i class="fas fa-sort-down ms-1"></i>
|
||||
}
|
||||
} else {
|
||||
<i class="fas fa-sort ms-1 text-muted"></i>
|
||||
}
|
||||
</a>
|
||||
</th>
|
||||
if data.ShowDataCenterColumn {
|
||||
<th class="text-dark">Data Centers</th>
|
||||
}
|
||||
<th class="text-dark">Actions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
for _, volume := range data.EcVolumes {
|
||||
<tr>
|
||||
<td>
|
||||
<span class="fw-bold">{fmt.Sprintf("%d", volume.VolumeID)}</span>
|
||||
</td>
|
||||
if data.ShowCollectionColumn {
|
||||
<td>
|
||||
if volume.Collection != "" {
|
||||
<a href={ templ.URL(fmt.Sprintf("/cluster/ec-shards?collection=%s", volume.Collection)) } class="text-decoration-none">
|
||||
<span class="badge bg-info text-white">{volume.Collection}</span>
|
||||
</a>
|
||||
} else {
|
||||
<a href={ templ.URL("/cluster/ec-shards?collection=default") } class="text-decoration-none">
|
||||
<span class="badge bg-secondary text-white">default</span>
|
||||
</a>
|
||||
}
|
||||
</td>
|
||||
}
|
||||
<td>
|
||||
<span class="badge bg-primary">{fmt.Sprintf("%d/%d", volume.TotalShards, erasure_coding.TotalShardsCount)}</span>
|
||||
</td>
|
||||
<td>
|
||||
@displayShardSizes(volume.ShardSizes)
|
||||
</td>
|
||||
<td>
|
||||
@displayVolumeDistribution(volume)
|
||||
</td>
|
||||
<td>
|
||||
@displayEcVolumeStatus(volume)
|
||||
</td>
|
||||
if data.ShowDataCenterColumn {
|
||||
<td>
|
||||
for i, dc := range volume.DataCenters {
|
||||
if i > 0 {
|
||||
<span>, </span>
|
||||
<th>
|
||||
<a href="#" onclick="sortBy('collection')" class="text-dark text-decoration-none">
|
||||
Collection
|
||||
if data.SortBy == "collection" {
|
||||
if data.SortOrder == "asc" {
|
||||
<i class="fas fa-sort-up ms-1"></i>
|
||||
} else {
|
||||
<i class="fas fa-sort-down ms-1"></i>
|
||||
}
|
||||
} else {
|
||||
<i class="fas fa-sort ms-1 text-muted"></i>
|
||||
}
|
||||
<span class="badge bg-primary text-white">{dc}</span>
|
||||
}
|
||||
</td>
|
||||
</a>
|
||||
</th>
|
||||
}
|
||||
<td>
|
||||
<div class="btn-group" role="group">
|
||||
<button type="button" class="btn btn-sm btn-outline-primary"
|
||||
onclick="showVolumeDetails(event)"
|
||||
data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
|
||||
title="View EC volume details">
|
||||
<i class="fas fa-info-circle"></i>
|
||||
</button>
|
||||
if !volume.IsComplete {
|
||||
<button type="button" class="btn btn-sm btn-outline-warning"
|
||||
onclick="repairVolume(event)"
|
||||
data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
|
||||
title="Repair missing shards">
|
||||
<i class="fas fa-wrench"></i>
|
||||
</button>
|
||||
<th>
|
||||
<a href="#" onclick="sortBy('total_shards')" class="text-dark text-decoration-none">
|
||||
Shard Count
|
||||
if data.SortBy == "total_shards" {
|
||||
if data.SortOrder == "asc" {
|
||||
<i class="fas fa-sort-up ms-1"></i>
|
||||
} else {
|
||||
<i class="fas fa-sort-down ms-1"></i>
|
||||
}
|
||||
} else {
|
||||
<i class="fas fa-sort ms-1 text-muted"></i>
|
||||
}
|
||||
</div>
|
||||
</td>
|
||||
</a>
|
||||
</th>
|
||||
<th class="text-dark">Shard Size</th>
|
||||
<th class="text-dark">Shard Locations</th>
|
||||
<th>
|
||||
<a href="#" onclick="sortBy('completeness')" class="text-dark text-decoration-none">
|
||||
Status
|
||||
if data.SortBy == "completeness" {
|
||||
if data.SortOrder == "asc" {
|
||||
<i class="fas fa-sort-up ms-1"></i>
|
||||
} else {
|
||||
<i class="fas fa-sort-down ms-1"></i>
|
||||
}
|
||||
} else {
|
||||
<i class="fas fa-sort ms-1 text-muted"></i>
|
||||
}
|
||||
</a>
|
||||
</th>
|
||||
if data.ShowDataCenterColumn {
|
||||
<th class="text-dark">Data Centers</th>
|
||||
}
|
||||
<th class="text-dark">Actions</th>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</thead>
|
||||
<tbody>
|
||||
for _, volume := range data.EcVolumes {
|
||||
<tr>
|
||||
<td>
|
||||
<strong>{fmt.Sprintf("%d", volume.VolumeID)}</strong>
|
||||
</td>
|
||||
if data.ShowCollectionColumn {
|
||||
<td>
|
||||
if volume.Collection != "" {
|
||||
<a href="/cluster/ec-shards?collection={volume.Collection}" class="text-decoration-none">
|
||||
<span class="badge bg-info text-white">{volume.Collection}</span>
|
||||
</a>
|
||||
} else {
|
||||
<a href="/cluster/ec-shards?collection=default" class="text-decoration-none">
|
||||
<span class="badge bg-secondary text-white">default</span>
|
||||
</a>
|
||||
}
|
||||
</td>
|
||||
}
|
||||
<td>
|
||||
<span class="badge bg-primary">{fmt.Sprintf("%d/14", volume.TotalShards)}</span>
|
||||
</td>
|
||||
<td>
|
||||
@displayShardSizes(volume.ShardSizes)
|
||||
</td>
|
||||
<td>
|
||||
@displayVolumeDistribution(volume)
|
||||
</td>
|
||||
<td>
|
||||
@displayEcVolumeStatus(volume)
|
||||
</td>
|
||||
if data.ShowDataCenterColumn {
|
||||
<td>
|
||||
for i, dc := range volume.DataCenters {
|
||||
if i > 0 {
|
||||
<span>, </span>
|
||||
}
|
||||
<span class="badge bg-primary text-white">{dc}</span>
|
||||
}
|
||||
</td>
|
||||
}
|
||||
<td>
|
||||
<div class="btn-group" role="group">
|
||||
<button type="button" class="btn btn-sm btn-outline-primary"
|
||||
onclick="showVolumeDetails(event)"
|
||||
data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
|
||||
title="View EC volume details">
|
||||
<i class="fas fa-info-circle"></i>
|
||||
</button>
|
||||
if !volume.IsComplete {
|
||||
<button type="button" class="btn btn-sm btn-outline-warning"
|
||||
onclick="repairVolume(event)"
|
||||
data-volume-id={ fmt.Sprintf("%d", volume.VolumeID) }
|
||||
title="Repair missing shards">
|
||||
<i class="fas fa-wrench"></i>
|
||||
</button>
|
||||
}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<!-- Pagination -->
|
||||
if data.TotalPages > 1 {
|
||||
<nav aria-label="EC Volumes pagination">
|
||||
<ul class="pagination justify-content-center">
|
||||
if data.Page > 1 {
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page="1">First</a>
|
||||
</li>
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>Previous</a>
|
||||
</li>
|
||||
}
|
||||
|
||||
for i := 1; i <= data.TotalPages; i++ {
|
||||
if i == data.Page {
|
||||
<li class="page-item active">
|
||||
<span class="page-link">{fmt.Sprintf("%d", i)}</span>
|
||||
</li>
|
||||
} else if i <= 3 || i > data.TotalPages-3 || (i >= data.Page-2 && i <= data.Page+2) {
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", i) }>{fmt.Sprintf("%d", i)}</a>
|
||||
</li>
|
||||
} else if i == 4 && data.Page > 6 {
|
||||
<li class="page-item disabled">
|
||||
<span class="page-link">...</span>
|
||||
</li>
|
||||
} else if i == data.TotalPages-3 && data.Page < data.TotalPages-5 {
|
||||
<li class="page-item disabled">
|
||||
<span class="page-link">...</span>
|
||||
</li>
|
||||
}
|
||||
}
|
||||
|
||||
if data.Page < data.TotalPages {
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>Next</a>
|
||||
</li>
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>Last</a>
|
||||
</li>
|
||||
}
|
||||
</ul>
|
||||
</nav>
|
||||
}
|
||||
</div>
|
||||
|
||||
<!-- Pagination -->
|
||||
if data.TotalPages > 1 {
|
||||
<nav aria-label="EC Volumes pagination">
|
||||
<ul class="pagination justify-content-center">
|
||||
if data.Page > 1 {
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page="1">
|
||||
<i class="fas fa-angle-double-left"></i>
|
||||
</a>
|
||||
</li>
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>
|
||||
<i class="fas fa-chevron-left"></i>
|
||||
</a>
|
||||
</li>
|
||||
}
|
||||
|
||||
<!-- First page -->
|
||||
if data.Page > 3 {
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page="1">1</a>
|
||||
</li>
|
||||
if data.Page > 4 {
|
||||
<li class="page-item disabled">
|
||||
<span class="page-link">...</span>
|
||||
</li>
|
||||
}
|
||||
}
|
||||
|
||||
<!-- Current page neighbors -->
|
||||
if data.Page > 1 {
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page-1) }>{fmt.Sprintf("%d", data.Page-1)}</a>
|
||||
</li>
|
||||
}
|
||||
|
||||
<li class="page-item active">
|
||||
<span class="page-link">{fmt.Sprintf("%d", data.Page)}</span>
|
||||
</li>
|
||||
|
||||
if data.Page < data.TotalPages {
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>{fmt.Sprintf("%d", data.Page+1)}</a>
|
||||
</li>
|
||||
}
|
||||
|
||||
<!-- Last page -->
|
||||
if data.Page < data.TotalPages-2 {
|
||||
if data.Page < data.TotalPages-3 {
|
||||
<li class="page-item disabled">
|
||||
<span class="page-link">...</span>
|
||||
</li>
|
||||
}
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>{fmt.Sprintf("%d", data.TotalPages)}</a>
|
||||
</li>
|
||||
}
|
||||
|
||||
if data.Page < data.TotalPages {
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.Page+1) }>
|
||||
<i class="fas fa-chevron-right"></i>
|
||||
</a>
|
||||
</li>
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="#" onclick="goToPage(event)" data-page={ fmt.Sprintf("%d", data.TotalPages) }>
|
||||
<i class="fas fa-angle-double-right"></i>
|
||||
</a>
|
||||
</li>
|
||||
}
|
||||
</ul>
|
||||
</nav>
|
||||
}
|
||||
|
||||
<!-- JavaScript -->
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
|
||||
<script>
|
||||
// Sorting functionality
|
||||
function sortBy(field) {
|
||||
const currentSort = new URLSearchParams(window.location.search).get('sort_by');
|
||||
const currentOrder = new URLSearchParams(window.location.search).get('sort_order') || 'asc';
|
||||
@@ -342,69 +335,65 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
|
||||
newOrder = 'desc';
|
||||
}
|
||||
|
||||
updateUrl({
|
||||
sort_by: field,
|
||||
sort_order: newOrder,
|
||||
page: 1
|
||||
});
|
||||
}
|
||||
|
||||
function goToPage(event) {
|
||||
event.preventDefault();
|
||||
const page = event.target.closest('a').getAttribute('data-page');
|
||||
updateUrl({ page: page });
|
||||
}
|
||||
|
||||
function changePageSize(newPageSize) {
|
||||
updateUrl({ page_size: newPageSize, page: 1 });
|
||||
}
|
||||
|
||||
function updateUrl(params) {
|
||||
const url = new URL(window.location);
|
||||
Object.keys(params).forEach(key => {
|
||||
if (params[key] != null) {
|
||||
url.searchParams.set(key, params[key]);
|
||||
} else {
|
||||
url.searchParams.delete(key);
|
||||
}
|
||||
});
|
||||
url.searchParams.set('sort_by', field);
|
||||
url.searchParams.set('sort_order', newOrder);
|
||||
url.searchParams.set('page', '1'); // Reset to first page
|
||||
window.location.href = url.toString();
|
||||
}
|
||||
|
||||
// Pagination functionality
|
||||
function goToPage(event) {
|
||||
event.preventDefault();
|
||||
const page = event.target.closest('a').getAttribute('data-page');
|
||||
const url = new URL(window.location);
|
||||
url.searchParams.set('page', page);
|
||||
window.location.href = url.toString();
|
||||
}
|
||||
|
||||
// Page size functionality
|
||||
function changePageSize(newPageSize) {
|
||||
const url = new URL(window.location);
|
||||
url.searchParams.set('page_size', newPageSize);
|
||||
url.searchParams.set('page', '1'); // Reset to first page when changing page size
|
||||
window.location.href = url.toString();
|
||||
}
|
||||
|
||||
// Volume details
|
||||
function showVolumeDetails(event) {
|
||||
const volumeId = event.target.closest('button').getAttribute('data-volume-id');
|
||||
window.location.href = `/cluster/ec-volumes/${volumeId}`;
|
||||
}
|
||||
|
||||
// Repair volume
|
||||
function repairVolume(event) {
|
||||
const volumeId = event.target.closest('button').getAttribute('data-volume-id');
|
||||
if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) {
|
||||
fetch(`/api/cluster/ec-volumes/${volumeId}/repair`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
}
|
||||
})
|
||||
.then(response => {
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
return response.json();
|
||||
})
|
||||
.then(data => {
|
||||
if (data && data.success) {
|
||||
alert('Repair initiated successfully');
|
||||
location.reload();
|
||||
} else {
|
||||
alert('Failed to initiate repair: ' + (data && data.error ? data.error : 'Unknown error'));
|
||||
}
|
||||
})
|
||||
.catch(error => {
|
||||
alert('Error: ' + error.message);
|
||||
});
|
||||
// TODO: Implement repair functionality
|
||||
alert('Repair functionality will be implemented soon.');
|
||||
}
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
}
|
||||
|
||||
// displayShardLocationsHTML renders shard locations as proper HTML
|
||||
templ displayShardLocationsHTML(shardLocations map[int]string) {
|
||||
if len(shardLocations) == 0 {
|
||||
<span class="text-muted">No shards</span>
|
||||
} else {
|
||||
for i, serverInfo := range groupShardsByServer(shardLocations) {
|
||||
if i > 0 {
|
||||
<br/>
|
||||
}
|
||||
<strong>
|
||||
<a href={ templ.URL("/cluster/volume-servers/" + serverInfo.Server) } class="text-primary text-decoration-none">
|
||||
{ serverInfo.Server }
|
||||
</a>:
|
||||
</strong> { serverInfo.ShardRanges }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// displayShardSizes renders shard sizes in a compact format
|
||||
@@ -450,7 +439,13 @@ func groupShardsByServer(shardLocations map[int]string) []ServerShardInfo {
|
||||
var serverInfos []ServerShardInfo
|
||||
for server, shards := range serverShards {
|
||||
// Sort shards for each server
|
||||
sort.Ints(shards)
|
||||
for i := 0; i < len(shards); i++ {
|
||||
for j := i + 1; j < len(shards); j++ {
|
||||
if shards[i] > shards[j] {
|
||||
shards[i], shards[j] = shards[j], shards[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Format shard ranges compactly
|
||||
shardRanges := formatShardRanges(shards)
|
||||
@@ -461,9 +456,56 @@ func groupShardsByServer(shardLocations map[int]string) []ServerShardInfo {
|
||||
}
|
||||
|
||||
// Sort by server name
|
||||
sort.Slice(serverInfos, func(i, j int) bool {
|
||||
return serverInfos[i].Server < serverInfos[j].Server
|
||||
})
|
||||
for i := 0; i < len(serverInfos); i++ {
|
||||
for j := i + 1; j < len(serverInfos); j++ {
|
||||
if serverInfos[i].Server > serverInfos[j].Server {
|
||||
serverInfos[i], serverInfos[j] = serverInfos[j], serverInfos[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return serverInfos
|
||||
}
|
||||
|
||||
// groupShardsByServerWithSizes groups shards by server and formats ranges with sizes
|
||||
func groupShardsByServerWithSizes(shardLocations map[int]string, shardSizes map[int]int64) []ServerShardInfo {
|
||||
if len(shardLocations) == 0 {
|
||||
return []ServerShardInfo{}
|
||||
}
|
||||
|
||||
// Group shards by server
|
||||
serverShards := make(map[string][]int)
|
||||
for shardId, server := range shardLocations {
|
||||
serverShards[server] = append(serverShards[server], shardId)
|
||||
}
|
||||
|
||||
var serverInfos []ServerShardInfo
|
||||
for server, shards := range serverShards {
|
||||
// Sort shards for each server
|
||||
for i := 0; i < len(shards); i++ {
|
||||
for j := i + 1; j < len(shards); j++ {
|
||||
if shards[i] > shards[j] {
|
||||
shards[i], shards[j] = shards[j], shards[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Format shard ranges compactly with sizes
|
||||
shardRanges := formatShardRangesWithSizes(shards, shardSizes)
|
||||
serverInfos = append(serverInfos, ServerShardInfo{
|
||||
Server: server,
|
||||
ShardRanges: shardRanges,
|
||||
})
|
||||
}
|
||||
|
||||
// Sort by server name
|
||||
for i := 0; i < len(serverInfos); i++ {
|
||||
for j := i + 1; j < len(serverInfos); j++ {
|
||||
if serverInfos[i].Server > serverInfos[j].Server {
|
||||
serverInfos[i], serverInfos[j] = serverInfos[j], serverInfos[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return serverInfos
|
||||
}
|
||||
@@ -502,6 +544,72 @@ func formatShardRanges(shards []int) string {
|
||||
return strings.Join(ranges, ",")
|
||||
}
|
||||
|
||||
// Helper function to format shard ranges with sizes (e.g., "0(1.2MB),1-3(2.5MB),7(800KB)")
|
||||
func formatShardRangesWithSizes(shards []int, shardSizes map[int]int64) string {
|
||||
if len(shards) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
var ranges []string
|
||||
start := shards[0]
|
||||
end := shards[0]
|
||||
var totalSize int64
|
||||
|
||||
for i := 1; i < len(shards); i++ {
|
||||
if shards[i] == end+1 {
|
||||
end = shards[i]
|
||||
totalSize += shardSizes[shards[i]]
|
||||
} else {
|
||||
// Add current range with size
|
||||
if start == end {
|
||||
size := shardSizes[start]
|
||||
if size > 0 {
|
||||
ranges = append(ranges, fmt.Sprintf("%d(%s)", start, bytesToHumanReadable(size)))
|
||||
} else {
|
||||
ranges = append(ranges, fmt.Sprintf("%d", start))
|
||||
}
|
||||
} else {
|
||||
// Calculate total size for the range
|
||||
rangeSize := shardSizes[start]
|
||||
for j := start + 1; j <= end; j++ {
|
||||
rangeSize += shardSizes[j]
|
||||
}
|
||||
if rangeSize > 0 {
|
||||
ranges = append(ranges, fmt.Sprintf("%d-%d(%s)", start, end, bytesToHumanReadable(rangeSize)))
|
||||
} else {
|
||||
ranges = append(ranges, fmt.Sprintf("%d-%d", start, end))
|
||||
}
|
||||
}
|
||||
start = shards[i]
|
||||
end = shards[i]
|
||||
totalSize = shardSizes[shards[i]]
|
||||
}
|
||||
}
|
||||
|
||||
// Add the last range
|
||||
if start == end {
|
||||
size := shardSizes[start]
|
||||
if size > 0 {
|
||||
ranges = append(ranges, fmt.Sprintf("%d(%s)", start, bytesToHumanReadable(size)))
|
||||
} else {
|
||||
ranges = append(ranges, fmt.Sprintf("%d", start))
|
||||
}
|
||||
} else {
|
||||
// Calculate total size for the range
|
||||
rangeSize := shardSizes[start]
|
||||
for j := start + 1; j <= end; j++ {
|
||||
rangeSize += shardSizes[j]
|
||||
}
|
||||
if rangeSize > 0 {
|
||||
ranges = append(ranges, fmt.Sprintf("%d-%d(%s)", start, end, bytesToHumanReadable(rangeSize)))
|
||||
} else {
|
||||
ranges = append(ranges, fmt.Sprintf("%d-%d", start, end))
|
||||
}
|
||||
}
|
||||
|
||||
return strings.Join(ranges, ",")
|
||||
}
|
||||
|
||||
// Helper function to convert bytes to human readable format
|
||||
func bytesToHumanReadable(bytes int64) string {
|
||||
const unit = 1024
|
||||
@@ -572,7 +680,13 @@ func formatIndividualShardSizes(shardSizes map[int]int64) string {
|
||||
var groupStrs []string
|
||||
for size, shardIds := range sizeGroups {
|
||||
// Sort shard IDs
|
||||
sort.Ints(shardIds)
|
||||
for i := 0; i < len(shardIds); i++ {
|
||||
for j := i + 1; j < len(shardIds); j++ {
|
||||
if shardIds[i] > shardIds[j] {
|
||||
shardIds[i], shardIds[j] = shardIds[j], shardIds[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var idRanges []string
|
||||
if len(shardIds) <= erasure_coding.ParityShardsCount {
|
||||
@@ -601,27 +715,18 @@ templ displayVolumeDistribution(volume dash.EcVolumeWithShards) {
|
||||
</div>
|
||||
}
|
||||
|
||||
// displayEcVolumeStatus shows an improved status display for EC volumes.
|
||||
// Status thresholds are based on EC recovery capability:
|
||||
// - Critical: More than DataShardsCount missing (data is unrecoverable)
|
||||
// - Degraded: More than half of DataShardsCount missing (high risk)
|
||||
// - Incomplete: More than half of ParityShardsCount missing (reduced redundancy)
|
||||
// - Minor Issues: Few shards missing (still fully recoverable)
|
||||
// displayEcVolumeStatus shows an improved status display for EC volumes
|
||||
templ displayEcVolumeStatus(volume dash.EcVolumeWithShards) {
|
||||
if volume.IsComplete {
|
||||
<span class="badge bg-success"><i class="fas fa-check me-1"></i>Complete</span>
|
||||
} else {
|
||||
if len(volume.MissingShards) > erasure_coding.DataShardsCount {
|
||||
// Unrecoverable: more shards missing than EC can reconstruct
|
||||
<span class="badge bg-danger"><i class="fas fa-skull me-1"></i>Critical ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
|
||||
} else if len(volume.MissingShards) > (erasure_coding.DataShardsCount/2) {
|
||||
// High risk: approaching unrecoverable state
|
||||
<span class="badge bg-warning"><i class="fas fa-exclamation-triangle me-1"></i>Degraded ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
|
||||
} else if len(volume.MissingShards) > (erasure_coding.ParityShardsCount/2) {
|
||||
// Reduced redundancy but still recoverable
|
||||
<span class="badge bg-warning"><i class="fas fa-info-circle me-1"></i>Incomplete ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
|
||||
} else {
|
||||
// Minor: few shards missing, fully recoverable with good margin
|
||||
<span class="badge bg-info"><i class="fas fa-info-circle me-1"></i>Minor Issues ({fmt.Sprintf("%d", len(volume.MissingShards))} missing)</span>
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -113,7 +113,6 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Owner</th>
|
||||
<th>Created</th>
|
||||
<th>Objects</th>
|
||||
<th>Size</th>
|
||||
@@ -127,21 +126,12 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
for _, bucket := range data.Buckets {
|
||||
<tr>
|
||||
<td>
|
||||
<a href={templ.SafeURL(fmt.Sprintf("/files?path=/buckets/%s", bucket.Name))}
|
||||
<a href={templ.SafeURL(fmt.Sprintf("/s3/buckets/%s", bucket.Name))}
|
||||
class="text-decoration-none">
|
||||
<i class="fas fa-cube me-2"></i>
|
||||
{bucket.Name}
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
if bucket.Owner != "" {
|
||||
<span class="badge bg-info">
|
||||
<i class="fas fa-user me-1"></i>{bucket.Owner}
|
||||
</span>
|
||||
} else {
|
||||
<span class="text-muted small">No owner</span>
|
||||
}
|
||||
</td>
|
||||
<td>{bucket.CreatedAt.Format("2006-01-02 15:04")}</td>
|
||||
<td>{fmt.Sprintf("%d", bucket.ObjectCount)}</td>
|
||||
<td>{formatBytes(bucket.Size)}</td>
|
||||
@@ -203,13 +193,6 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
title="View Details">
|
||||
<i class="fas fa-eye"></i>
|
||||
</button>
|
||||
<button type="button"
|
||||
class="btn btn-outline-info btn-sm owner-btn"
|
||||
data-bucket-name={bucket.Name}
|
||||
data-current-owner={bucket.Owner}
|
||||
title="Manage Owner">
|
||||
<i class="fas fa-user-edit"></i>
|
||||
</button>
|
||||
<button type="button"
|
||||
class="btn btn-outline-warning btn-sm quota-btn"
|
||||
data-bucket-name={bucket.Name}
|
||||
@@ -230,7 +213,7 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
}
|
||||
if len(data.Buckets) == 0 {
|
||||
<tr>
|
||||
<td colspan="9" class="text-center text-muted py-4">
|
||||
<td colspan="8" class="text-center text-muted py-4">
|
||||
<i class="fas fa-cube fa-3x mb-3 text-muted"></i>
|
||||
<div>
|
||||
<h5>No Object Store buckets found</h5>
|
||||
@@ -285,17 +268,6 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
Bucket names must be between 3 and 63 characters, contain only lowercase letters, numbers, dots, and hyphens.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="bucketOwner" class="form-label">Owner (Optional)</label>
|
||||
<select class="form-select" id="bucketOwner" name="owner">
|
||||
<option value="">No owner (admin-only access)</option>
|
||||
<!-- Options will be populated dynamically when modal opens -->
|
||||
</select>
|
||||
<div class="form-text">
|
||||
The S3 identity that owns this bucket. Non-admin users can only access buckets they own.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<div class="form-check">
|
||||
@@ -509,68 +481,9 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Manage Owner Modal -->
|
||||
<div class="modal fade" id="manageOwnerModal" tabindex="-1" aria-labelledby="manageOwnerModalLabel" aria-hidden="true">
|
||||
<div class="modal-dialog">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header">
|
||||
<h5 class="modal-title" id="manageOwnerModalLabel">
|
||||
<i class="fas fa-user-edit me-2"></i>Manage Bucket Owner
|
||||
</h5>
|
||||
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
|
||||
</div>
|
||||
<form id="ownerForm">
|
||||
<div class="modal-body">
|
||||
<div class="mb-3">
|
||||
<label class="form-label">Bucket Name</label>
|
||||
<input type="text" class="form-control" id="ownerBucketName" readonly>
|
||||
</div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label for="bucketOwnerSelect" class="form-label">Owner</label>
|
||||
<select class="form-select" id="bucketOwnerSelect" name="owner">
|
||||
<option value="">No owner (admin-only access)</option>
|
||||
<!-- Options will be populated dynamically -->
|
||||
</select>
|
||||
<div class="form-text">
|
||||
Select the S3 identity that owns this bucket. Non-admin users can only access buckets they own.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="ownerLoadingSpinner" class="text-center py-2" style="display: none;">
|
||||
<div class="spinner-border spinner-border-sm text-primary" role="status">
|
||||
<span class="visually-hidden">Loading users...</span>
|
||||
</div>
|
||||
<span class="ms-2">Loading users...</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
|
||||
<button type="submit" class="btn btn-info">
|
||||
<i class="fas fa-save me-1"></i>Update Owner
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- JavaScript for bucket management -->
|
||||
<script>
|
||||
// Global state (shared between DOMContentLoaded handlers and global functions)
|
||||
let deleteModalInstance = null;
|
||||
let quotaModalInstance = null;
|
||||
let ownerModalInstance = null;
|
||||
let detailsModalInstance = null;
|
||||
let cachedUsers = null;
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
// Initialize modal instances once (reuse with show/hide)
|
||||
deleteModalInstance = new bootstrap.Modal(document.getElementById('deleteBucketModal'));
|
||||
quotaModalInstance = new bootstrap.Modal(document.getElementById('manageQuotaModal'));
|
||||
ownerModalInstance = new bootstrap.Modal(document.getElementById('manageOwnerModal'));
|
||||
detailsModalInstance = new bootstrap.Modal(document.getElementById('bucketDetailsModal'));
|
||||
|
||||
const quotaCheckbox = document.getElementById('enableQuota');
|
||||
const quotaSettings = document.getElementById('quotaSettings');
|
||||
const versioningCheckbox = document.getElementById('enableVersioning');
|
||||
@@ -604,32 +517,6 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
defaultRetentionSettings.style.display = this.checked ? 'block' : 'none';
|
||||
});
|
||||
|
||||
// Populate owner dropdown when create bucket modal opens
|
||||
document.getElementById('createBucketModal').addEventListener('show.bs.modal', async function() {
|
||||
const ownerSelect = document.getElementById('bucketOwner');
|
||||
|
||||
// Only fetch if not already populated
|
||||
if (ownerSelect.options.length <= 1) {
|
||||
try {
|
||||
const response = await fetch('/api/users');
|
||||
const data = await response.json();
|
||||
const users = data.users || [];
|
||||
|
||||
users.forEach(user => {
|
||||
const option = document.createElement('option');
|
||||
option.value = user.username;
|
||||
option.textContent = user.username;
|
||||
ownerSelect.appendChild(option);
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching users for owner dropdown:', error);
|
||||
// Reset to default state on error - user can still create bucket without owner
|
||||
ownerSelect.innerHTML = '<option value="">No owner (admin-only access)</option>';
|
||||
ownerSelect.selectedIndex = 0;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Handle form submission
|
||||
createBucketForm.addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
@@ -637,7 +524,6 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
const formData = new FormData(this);
|
||||
const data = {
|
||||
name: formData.get('name'),
|
||||
owner: formData.get('owner') || '',
|
||||
region: formData.get('region') || '',
|
||||
quota_size: quotaCheckbox.checked ? parseInt(formData.get('quota_size')) || 0 : 0,
|
||||
quota_unit: formData.get('quota_unit') || 'MB',
|
||||
@@ -683,19 +569,41 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
});
|
||||
|
||||
// Handle delete bucket
|
||||
const deleteForm = document.getElementById('deleteBucketModal');
|
||||
let deleteModalInstance = null;
|
||||
document.querySelectorAll('.delete-bucket-btn').forEach(button => {
|
||||
button.addEventListener('click', function() {
|
||||
const bucketName = this.dataset.bucketName;
|
||||
document.getElementById('deleteBucketName').textContent = bucketName;
|
||||
// Store bucket name on modal element instead of global window property
|
||||
deleteForm.dataset.bucketName = bucketName;
|
||||
window.currentBucketToDelete = bucketName;
|
||||
|
||||
// Dispose of existing modal instance if it exists
|
||||
if (deleteModalInstance) {
|
||||
deleteModalInstance.dispose();
|
||||
}
|
||||
|
||||
// Create new modal instance
|
||||
deleteModalInstance = new bootstrap.Modal(document.getElementById('deleteBucketModal'));
|
||||
deleteModalInstance.show();
|
||||
});
|
||||
});
|
||||
|
||||
// Add event listener to properly dispose of delete modal when hidden
|
||||
document.getElementById('deleteBucketModal').addEventListener('hidden.bs.modal', function() {
|
||||
if (deleteModalInstance) {
|
||||
deleteModalInstance.dispose();
|
||||
deleteModalInstance = null;
|
||||
}
|
||||
// Force remove any remaining backdrops
|
||||
document.querySelectorAll('.modal-backdrop').forEach(backdrop => {
|
||||
backdrop.remove();
|
||||
});
|
||||
// Ensure body classes are removed
|
||||
document.body.classList.remove('modal-open');
|
||||
document.body.style.removeProperty('padding-right');
|
||||
});
|
||||
|
||||
// Handle quota management
|
||||
const quotaForm = document.getElementById('quotaForm');
|
||||
let quotaModalInstance = null;
|
||||
document.querySelectorAll('.quota-btn').forEach(button => {
|
||||
button.addEventListener('click', function() {
|
||||
const bucketName = this.dataset.bucketName;
|
||||
@@ -709,8 +617,15 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
// Toggle quota size settings
|
||||
document.getElementById('quotaSizeSettings').style.display = quotaEnabled ? 'block' : 'none';
|
||||
|
||||
// Store bucket name on form element instead of global window property
|
||||
quotaForm.dataset.bucketName = bucketName;
|
||||
window.currentBucketToUpdate = bucketName;
|
||||
|
||||
// Dispose of existing modal instance if it exists
|
||||
if (quotaModalInstance) {
|
||||
quotaModalInstance.dispose();
|
||||
}
|
||||
|
||||
// Create new modal instance
|
||||
quotaModalInstance = new bootstrap.Modal(document.getElementById('manageQuotaModal'));
|
||||
quotaModalInstance.show();
|
||||
});
|
||||
});
|
||||
@@ -734,9 +649,6 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
document.getElementById('quotaForm').addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
|
||||
const bucketName = this.dataset.bucketName;
|
||||
if (!bucketName) return;
|
||||
|
||||
const formData = new FormData(this);
|
||||
const enabled = document.getElementById('quotaEnabled').checked;
|
||||
const data = {
|
||||
@@ -745,7 +657,7 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
quota_enabled: enabled
|
||||
};
|
||||
|
||||
fetch(`/api/s3/buckets/${bucketName}/quota`, {
|
||||
fetch(`/api/s3/buckets/${window.currentBucketToUpdate}/quota`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
@@ -776,97 +688,8 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
document.getElementById('quotaSizeSettings').style.display = this.checked ? 'block' : 'none';
|
||||
});
|
||||
|
||||
// Handle owner management
|
||||
const ownerForm = document.getElementById('ownerForm');
|
||||
document.querySelectorAll('.owner-btn').forEach(button => {
|
||||
button.addEventListener('click', async function() {
|
||||
const bucketName = this.dataset.bucketName;
|
||||
const currentOwner = this.dataset.currentOwner || '';
|
||||
|
||||
document.getElementById('ownerBucketName').value = bucketName;
|
||||
// Store bucket name on form element instead of global window property
|
||||
ownerForm.dataset.bucketName = bucketName;
|
||||
|
||||
// Show loading spinner
|
||||
document.getElementById('ownerLoadingSpinner').style.display = 'block';
|
||||
document.getElementById('bucketOwnerSelect').disabled = true;
|
||||
|
||||
ownerModalInstance.show();
|
||||
|
||||
// Fetch users if not cached
|
||||
try {
|
||||
if (!cachedUsers) {
|
||||
const response = await fetch('/api/users');
|
||||
const data = await response.json();
|
||||
cachedUsers = data.users || [];
|
||||
}
|
||||
|
||||
// Populate the select dropdown
|
||||
const select = document.getElementById('bucketOwnerSelect');
|
||||
select.innerHTML = '<option value="">No owner (admin-only access)</option>';
|
||||
|
||||
cachedUsers.forEach(user => {
|
||||
const option = document.createElement('option');
|
||||
option.value = user.username;
|
||||
option.textContent = user.username;
|
||||
if (user.username === currentOwner) {
|
||||
option.selected = true;
|
||||
}
|
||||
select.appendChild(option);
|
||||
});
|
||||
|
||||
select.disabled = false;
|
||||
} catch (error) {
|
||||
console.error('Error fetching users:', error);
|
||||
alert('Error loading users: ' + error.message);
|
||||
// Re-enable select and reset to default on error
|
||||
const select = document.getElementById('bucketOwnerSelect');
|
||||
select.innerHTML = '<option value="">No owner (admin-only access)</option>';
|
||||
select.selectedIndex = 0;
|
||||
select.disabled = false;
|
||||
} finally {
|
||||
document.getElementById('ownerLoadingSpinner').style.display = 'none';
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// Handle owner form submission
|
||||
document.getElementById('ownerForm').addEventListener('submit', function(e) {
|
||||
e.preventDefault();
|
||||
|
||||
const bucketName = this.dataset.bucketName;
|
||||
if (!bucketName) return;
|
||||
|
||||
const owner = document.getElementById('bucketOwnerSelect').value;
|
||||
const data = { owner: owner };
|
||||
|
||||
fetch(`/api/s3/buckets/${bucketName}/owner`, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(data)
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
if (data.error) {
|
||||
alert('Error updating owner: ' + data.error);
|
||||
} else {
|
||||
alert('Bucket owner updated successfully!');
|
||||
// Properly close the modal before reloading
|
||||
if (ownerModalInstance) {
|
||||
ownerModalInstance.hide();
|
||||
}
|
||||
setTimeout(() => location.reload(), 500);
|
||||
}
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('Error:', error);
|
||||
alert('Error updating owner: ' + error.message);
|
||||
});
|
||||
});
|
||||
|
||||
// Handle view details button
|
||||
let detailsModalInstance = null;
|
||||
document.querySelectorAll('.view-details-btn').forEach(button => {
|
||||
button.addEventListener('click', function() {
|
||||
const bucketName = this.dataset.bucketName;
|
||||
@@ -884,6 +707,13 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
'<div class="mt-2">Loading bucket details...</div>' +
|
||||
'</div>';
|
||||
|
||||
// Dispose of existing modal instance if it exists
|
||||
if (detailsModalInstance) {
|
||||
detailsModalInstance.dispose();
|
||||
}
|
||||
|
||||
// Create new modal instance
|
||||
detailsModalInstance = new bootstrap.Modal(document.getElementById('bucketDetailsModal'));
|
||||
detailsModalInstance.show();
|
||||
|
||||
// Fetch bucket details
|
||||
@@ -910,10 +740,25 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Add event listener to properly dispose of details modal when hidden
|
||||
document.getElementById('bucketDetailsModal').addEventListener('hidden.bs.modal', function() {
|
||||
if (detailsModalInstance) {
|
||||
detailsModalInstance.dispose();
|
||||
detailsModalInstance = null;
|
||||
}
|
||||
// Force remove any remaining backdrops
|
||||
document.querySelectorAll('.modal-backdrop').forEach(backdrop => {
|
||||
backdrop.remove();
|
||||
});
|
||||
// Ensure body classes are removed
|
||||
document.body.classList.remove('modal-open');
|
||||
document.body.style.removeProperty('padding-right');
|
||||
});
|
||||
});
|
||||
|
||||
function deleteBucket() {
|
||||
const bucketName = document.getElementById('deleteBucketModal').dataset.bucketName;
|
||||
const bucketName = window.currentBucketToDelete;
|
||||
if (!bucketName) return;
|
||||
|
||||
fetch(`/api/s3/buckets/${bucketName}`, {
|
||||
@@ -941,16 +786,6 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
function displayBucketDetails(data) {
|
||||
const bucket = data.bucket;
|
||||
const objects = data.objects || [];
|
||||
|
||||
// Helper function to escape HTML to prevent XSS
|
||||
function escapeHtml(v) {
|
||||
return String(v ?? '')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, ''');
|
||||
}
|
||||
|
||||
// Helper function to format bytes
|
||||
function formatBytes(bytes) {
|
||||
@@ -983,10 +818,10 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
'<tbody>' +
|
||||
objects.map(obj =>
|
||||
'<tr>' +
|
||||
'<td><i class="fas fa-file me-1"></i>' + escapeHtml(obj.key) + '</td>' +
|
||||
'<td><i class="fas fa-file me-1"></i>' + obj.key + '</td>' +
|
||||
'<td>' + formatBytes(obj.size) + '</td>' +
|
||||
'<td>' + formatDate(obj.last_modified) + '</td>' +
|
||||
'<td><span class="badge bg-primary">' + escapeHtml(obj.storage_class) + '</span></td>' +
|
||||
'<td><span class="badge bg-primary">' + obj.storage_class + '</span></td>' +
|
||||
'</tr>'
|
||||
).join('') +
|
||||
'</tbody>' +
|
||||
@@ -1005,11 +840,7 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
'<table class="table table-sm">' +
|
||||
'<tr>' +
|
||||
'<td><strong>Name:</strong></td>' +
|
||||
'<td>' + escapeHtml(bucket.name) + '</td>' +
|
||||
'</tr>' +
|
||||
'<tr>' +
|
||||
'<td><strong>Owner:</strong></td>' +
|
||||
'<td>' + (bucket.owner ? '<span class="badge bg-info"><i class="fas fa-user me-1"></i>' + escapeHtml(bucket.owner) + '</span>' : '<span class="text-muted">No owner (admin-only)</span>') + '</td>' +
|
||||
'<td>' + bucket.name + '</td>' +
|
||||
'</tr>' +
|
||||
'<tr>' +
|
||||
'<td><strong>Created:</strong></td>' +
|
||||
@@ -1055,7 +886,7 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
'<td>' +
|
||||
(bucket.object_lock_enabled ?
|
||||
'<span class="badge bg-warning"><i class="fas fa-lock me-1"></i>Enabled</span>' +
|
||||
'<br><small class="text-muted">' + escapeHtml(bucket.object_lock_mode) + ' • ' + bucket.object_lock_duration + ' days</small>' :
|
||||
'<br><small class="text-muted">' + bucket.object_lock_mode + ' • ' + bucket.object_lock_duration + ' days</small>' :
|
||||
'<span class="badge bg-secondary"><i class="fas fa-unlock me-1"></i>Disabled</span>'
|
||||
) +
|
||||
'</td>' +
|
||||
@@ -1075,45 +906,26 @@ templ S3Buckets(data dash.S3BucketsData) {
|
||||
}
|
||||
|
||||
function exportBucketList() {
|
||||
// RFC 4180 compliant CSV escaping: escape double quotes by doubling them
|
||||
function escapeCsvField(value) {
|
||||
const str = String(value ?? '');
|
||||
// If the field contains comma, double quote, or newline, wrap in quotes and escape internal quotes
|
||||
if (str.includes(',') || str.includes('"') || str.includes('\n') || str.includes('\r')) {
|
||||
return '"' + str.replace(/"/g, '""') + '"';
|
||||
}
|
||||
return '"' + str + '"';
|
||||
}
|
||||
|
||||
// Simple CSV export
|
||||
const buckets = Array.from(document.querySelectorAll('#bucketsTable tbody tr')).map(row => {
|
||||
const cells = row.querySelectorAll('td');
|
||||
if (cells.length > 1) {
|
||||
return {
|
||||
name: cells[0].textContent.trim(),
|
||||
owner: cells[1].textContent.trim(),
|
||||
created: cells[2].textContent.trim(),
|
||||
objects: cells[3].textContent.trim(),
|
||||
size: cells[4].textContent.trim(),
|
||||
quota: cells[5].textContent.trim(),
|
||||
versioning: cells[6].textContent.trim(),
|
||||
objectLock: cells[7].textContent.trim()
|
||||
created: cells[1].textContent.trim(),
|
||||
objects: cells[2].textContent.trim(),
|
||||
size: cells[3].textContent.trim(),
|
||||
quota: cells[4].textContent.trim(),
|
||||
versioning: cells[5].textContent.trim(),
|
||||
objectLock: cells[6].textContent.trim()
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}).filter(bucket => bucket !== null);
|
||||
|
||||
const csvContent = "data:text/csv;charset=utf-8," +
|
||||
"Name,Owner,Created,Objects,Size,Quota,Versioning,Object Lock\n" +
|
||||
buckets.map(b => [
|
||||
escapeCsvField(b.name),
|
||||
escapeCsvField(b.owner),
|
||||
escapeCsvField(b.created),
|
||||
escapeCsvField(b.objects),
|
||||
escapeCsvField(b.size),
|
||||
escapeCsvField(b.quota),
|
||||
escapeCsvField(b.versioning),
|
||||
escapeCsvField(b.objectLock)
|
||||
].join(',')).join("\n");
|
||||
"Name,Created,Objects,Size,Quota,Versioning,Object Lock\n" +
|
||||
buckets.map(b => '"' + b.name + '","' + b.created + '","' + b.objects + '","' + b.size + '","' + b.quota + '","' + b.versioning + '","' + b.objectLock + '"').join("\n");
|
||||
|
||||
const encodedUri = encodeURI(csvContent);
|
||||
const link = document.createElement("a");
|
||||
|
||||
File diff suppressed because one or more lines are too long
+14
-39
@@ -3,8 +3,6 @@ package cluster
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
@@ -38,11 +36,10 @@ type LiveLock struct {
|
||||
hostFiler pb.ServerAddress
|
||||
cancelCh chan struct{}
|
||||
grpcDialOption grpc.DialOption
|
||||
isLocked int32 // 0 = unlocked, 1 = locked; use atomic operations
|
||||
isLocked bool
|
||||
self string
|
||||
lc *LockClient
|
||||
owner string
|
||||
lockTTL time.Duration
|
||||
}
|
||||
|
||||
// NewShortLivedLock creates a lock with a 5-second duration
|
||||
@@ -61,24 +58,17 @@ func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLoc
|
||||
}
|
||||
|
||||
// StartLongLivedLock starts a goroutine to lock the key and returns immediately.
|
||||
// lockTTL specifies how long the lock should be held. The renewal interval is
|
||||
// automatically derived as lockTTL / 2 to ensure timely renewals.
|
||||
func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string), lockTTL time.Duration) (lock *LiveLock) {
|
||||
func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string)) (lock *LiveLock) {
|
||||
lock = &LiveLock{
|
||||
key: key,
|
||||
hostFiler: lc.seedFiler,
|
||||
cancelCh: make(chan struct{}),
|
||||
expireAtNs: time.Now().Add(lockTTL).UnixNano(),
|
||||
expireAtNs: time.Now().Add(lock_manager.LiveLockTTL).UnixNano(),
|
||||
grpcDialOption: lc.grpcDialOption,
|
||||
self: owner,
|
||||
lc: lc,
|
||||
lockTTL: lockTTL,
|
||||
}
|
||||
if lock.lockTTL == 0 {
|
||||
lock.lockTTL = lock_manager.LiveLockTTL
|
||||
}
|
||||
go func() {
|
||||
renewInterval := lock.lockTTL / 2
|
||||
isLocked := false
|
||||
lockOwner := ""
|
||||
for {
|
||||
@@ -91,15 +81,13 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
|
||||
}
|
||||
|
||||
if isLocked {
|
||||
if err := lock.AttemptToLock(lock.lockTTL); err != nil {
|
||||
if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err != nil {
|
||||
glog.V(0).Infof("Lost lock %s: %v", key, err)
|
||||
isLocked = false
|
||||
atomic.StoreInt32(&lock.isLocked, 0)
|
||||
}
|
||||
} else {
|
||||
if err := lock.AttemptToLock(lock.lockTTL); err == nil {
|
||||
if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err == nil {
|
||||
isLocked = true
|
||||
// Note: AttemptToLock already sets lock.isLocked atomically on success
|
||||
}
|
||||
}
|
||||
if lockOwner != lock.LockOwner() && lock.LockOwner() != "" {
|
||||
@@ -111,11 +99,7 @@ func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerCh
|
||||
case <-lock.cancelCh:
|
||||
return
|
||||
default:
|
||||
if isLocked {
|
||||
time.Sleep(renewInterval)
|
||||
} else {
|
||||
time.Sleep(5 * renewInterval)
|
||||
}
|
||||
time.Sleep(lock_manager.RenewInterval)
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -142,28 +126,24 @@ func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
|
||||
return err
|
||||
}
|
||||
if errorMessage != "" {
|
||||
if strings.Contains(errorMessage, "lock already owned") {
|
||||
glog.V(3).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
|
||||
} else {
|
||||
glog.V(2).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
|
||||
}
|
||||
glog.V(1).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
|
||||
time.Sleep(time.Second)
|
||||
return fmt.Errorf("%v", errorMessage)
|
||||
}
|
||||
if atomic.LoadInt32(&lock.isLocked) == 0 {
|
||||
if !lock.isLocked {
|
||||
// Only log when transitioning from unlocked to locked
|
||||
glog.V(1).Infof("LOCK: Successfully acquired key=%s owner=%s", lock.key, lock.self)
|
||||
}
|
||||
atomic.StoreInt32(&lock.isLocked, 1)
|
||||
lock.isLocked = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (lock *LiveLock) StopShortLivedLock() error {
|
||||
if atomic.LoadInt32(&lock.isLocked) == 0 {
|
||||
if !lock.isLocked {
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
atomic.StoreInt32(&lock.isLocked, 0)
|
||||
lock.isLocked = false
|
||||
}()
|
||||
return pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
_, err := client.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
|
||||
@@ -223,7 +203,7 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
|
||||
errorMessage = resp.Error
|
||||
if resp.LockHostMovedTo != "" && resp.LockHostMovedTo != string(previousHostFiler) {
|
||||
// Only log if the host actually changed
|
||||
glog.V(2).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
|
||||
glog.V(1).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
|
||||
lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
|
||||
lock.lc.seedFiler = lock.hostFiler
|
||||
} else if resp.LockHostMovedTo != "" {
|
||||
@@ -231,12 +211,12 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
|
||||
}
|
||||
if resp.LockOwner != "" && resp.LockOwner != previousOwner {
|
||||
// Only log if the owner actually changed
|
||||
glog.V(2).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
|
||||
glog.V(1).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
|
||||
lock.owner = resp.LockOwner
|
||||
} else if resp.LockOwner != "" {
|
||||
lock.owner = resp.LockOwner
|
||||
} else if previousOwner != "" {
|
||||
glog.V(2).Infof("LOCK: Owner cleared for key=%s", lock.key)
|
||||
glog.V(1).Infof("LOCK: Owner cleared for key=%s", lock.key)
|
||||
lock.owner = ""
|
||||
}
|
||||
}
|
||||
@@ -248,8 +228,3 @@ func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, e
|
||||
func (lock *LiveLock) LockOwner() string {
|
||||
return lock.owner
|
||||
}
|
||||
|
||||
// IsLocked returns true if this instance currently holds the lock
|
||||
func (lock *LiveLock) IsLocked() bool {
|
||||
return atomic.LoadInt32(&lock.isLocked) == 1
|
||||
}
|
||||
|
||||
@@ -32,7 +32,6 @@ var Commands = []*Command{
|
||||
cmdIam,
|
||||
cmdMaster,
|
||||
cmdMasterFollower,
|
||||
cmdMini,
|
||||
cmdMount,
|
||||
cmdMqAgent,
|
||||
cmdMqBroker,
|
||||
|
||||
+33
-37
@@ -42,40 +42,39 @@ var (
|
||||
)
|
||||
|
||||
type FilerOptions struct {
|
||||
masters *pb.ServerDiscovery
|
||||
mastersString *string
|
||||
ip *string
|
||||
bindIp *string
|
||||
port *int
|
||||
portGrpc *int
|
||||
publicPort *int
|
||||
filerGroup *string
|
||||
collection *string
|
||||
defaultReplicaPlacement *string
|
||||
disableDirListing *bool
|
||||
maxMB *int
|
||||
dirListingLimit *int
|
||||
dataCenter *string
|
||||
rack *string
|
||||
enableNotification *bool
|
||||
disableHttp *bool
|
||||
cipher *bool
|
||||
metricsHttpPort *int
|
||||
metricsHttpIp *string
|
||||
saveToFilerLimit *int
|
||||
defaultLevelDbDirectory *string
|
||||
concurrentUploadLimitMB *int
|
||||
concurrentFileUploadLimit *int
|
||||
debug *bool
|
||||
debugPort *int
|
||||
localSocket *string
|
||||
showUIDirectoryDelete *bool
|
||||
downloadMaxMBps *int
|
||||
diskType *string
|
||||
allowedOrigins *string
|
||||
exposeDirectoryData *bool
|
||||
tusBasePath *string
|
||||
certProvider certprovider.Provider
|
||||
masters *pb.ServerDiscovery
|
||||
mastersString *string
|
||||
ip *string
|
||||
bindIp *string
|
||||
port *int
|
||||
portGrpc *int
|
||||
publicPort *int
|
||||
filerGroup *string
|
||||
collection *string
|
||||
defaultReplicaPlacement *string
|
||||
disableDirListing *bool
|
||||
maxMB *int
|
||||
dirListingLimit *int
|
||||
dataCenter *string
|
||||
rack *string
|
||||
enableNotification *bool
|
||||
disableHttp *bool
|
||||
cipher *bool
|
||||
metricsHttpPort *int
|
||||
metricsHttpIp *string
|
||||
saveToFilerLimit *int
|
||||
defaultLevelDbDirectory *string
|
||||
concurrentUploadLimitMB *int
|
||||
concurrentFileUploadLimit *int
|
||||
debug *bool
|
||||
debugPort *int
|
||||
localSocket *string
|
||||
showUIDirectoryDelete *bool
|
||||
downloadMaxMBps *int
|
||||
diskType *string
|
||||
allowedOrigins *string
|
||||
exposeDirectoryData *bool
|
||||
certProvider certprovider.Provider
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -110,7 +109,6 @@ func init() {
|
||||
f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
|
||||
f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI")
|
||||
f.tusBasePath = cmdFiler.Flag.String("tusBasePath", "/.tus", "TUS resumable upload endpoint base path (e.g., /.tus)")
|
||||
|
||||
// start s3 on filer
|
||||
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
|
||||
@@ -136,7 +134,6 @@ func init() {
|
||||
filerS3Options.idleTimeout = cmdFiler.Flag.Int("s3.idleTimeout", 120, "connection idle seconds")
|
||||
filerS3Options.concurrentUploadLimitMB = cmdFiler.Flag.Int("s3.concurrentUploadLimitMB", 0, "limit total concurrent upload size for S3, 0 means unlimited")
|
||||
filerS3Options.concurrentFileUploadLimit = cmdFiler.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads for S3, 0 means unlimited")
|
||||
filerS3Options.enableIam = cmdFiler.Flag.Bool("s3.iam", true, "enable embedded IAM API on the same S3 port")
|
||||
|
||||
// start webdav on filer
|
||||
filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
|
||||
@@ -344,7 +341,6 @@ func (fo *FilerOptions) startFiler() {
|
||||
DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024,
|
||||
DiskType: *fo.diskType,
|
||||
AllowedOrigins: strings.Split(*fo.allowedOrigins, ","),
|
||||
TusBasePath: *fo.tusBasePath,
|
||||
})
|
||||
if nfs_err != nil {
|
||||
glog.Fatalf("Filer startup error: %v", nfs_err)
|
||||
|
||||
@@ -53,8 +53,6 @@ type SyncOptions struct {
|
||||
bDoDeleteFiles *bool
|
||||
clientId int32
|
||||
clientEpoch atomic.Int32
|
||||
debug *bool
|
||||
debugPort *int
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -62,22 +60,10 @@ const (
|
||||
DefaultConcurrencyLimit = 32
|
||||
)
|
||||
|
||||
// syncState tracks the current sync state for graceful shutdown checkpoint saving
|
||||
type syncState struct {
|
||||
processor *MetadataProcessor
|
||||
grpcDialOption grpc.DialOption
|
||||
targetFiler pb.ServerAddress
|
||||
sourcePath string
|
||||
sourceFilerSignature int32
|
||||
}
|
||||
|
||||
var (
|
||||
syncOptions SyncOptions
|
||||
syncCpuProfile *string
|
||||
syncMemProfile *string
|
||||
// atomic pointers to current sync states for graceful shutdown
|
||||
syncStateA2B atomic.Pointer[syncState]
|
||||
syncStateB2A atomic.Pointer[syncState]
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -110,8 +96,6 @@ func init() {
|
||||
syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
|
||||
syncOptions.aDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("a.doDeleteFiles", true, "delete and update files when synchronizing on filer A")
|
||||
syncOptions.bDoDeleteFiles = cmdFilerSynchronize.Flag.Bool("b.doDeleteFiles", true, "delete and update files when synchronizing on filer B")
|
||||
syncOptions.debug = cmdFilerSynchronize.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
|
||||
syncOptions.debugPort = cmdFilerSynchronize.Flag.Int("debug.port", 6060, "http port for debugging")
|
||||
syncOptions.clientId = util.RandomInt32()
|
||||
}
|
||||
|
||||
@@ -134,9 +118,6 @@ var cmdFilerSynchronize = &Command{
|
||||
}
|
||||
|
||||
func runFilerSynchronize(cmd *Command, args []string) bool {
|
||||
if *syncOptions.debug {
|
||||
grace.StartDebugServer(*syncOptions.debugPort)
|
||||
}
|
||||
|
||||
util.LoadSecurityConfiguration()
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
|
||||
@@ -162,27 +143,6 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// register graceful shutdown hook to save checkpoints
|
||||
grace.OnInterrupt(func() {
|
||||
saveCheckpoint := func(name string, state *syncState) {
|
||||
if state == nil || state.processor == nil {
|
||||
return
|
||||
}
|
||||
offsetTsNs := state.processor.processedTsWatermark.Load()
|
||||
if offsetTsNs == 0 {
|
||||
return
|
||||
}
|
||||
if err := setOffset(state.grpcDialOption, state.targetFiler, getSignaturePrefixByPath(state.sourcePath), state.sourceFilerSignature, offsetTsNs); err != nil {
|
||||
glog.Errorf("failed to save checkpoint for %s on shutdown: %v", name, err)
|
||||
} else {
|
||||
glog.V(0).Infof("saved checkpoint for %s on shutdown: %v", name, time.Unix(0, offsetTsNs))
|
||||
}
|
||||
}
|
||||
|
||||
saveCheckpoint("A->B", syncStateA2B.Load())
|
||||
saveCheckpoint("B->A", syncStateB2A.Load())
|
||||
})
|
||||
|
||||
go func() {
|
||||
// a->b
|
||||
// set synchronization start timestamp to offset
|
||||
@@ -212,8 +172,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
|
||||
*syncOptions.concurrency,
|
||||
*syncOptions.bDoDeleteFiles,
|
||||
aFilerSignature,
|
||||
bFilerSignature,
|
||||
&syncStateA2B)
|
||||
bFilerSignature)
|
||||
if err != nil {
|
||||
glog.Errorf("sync from %s to %s: %v", *syncOptions.filerA, *syncOptions.filerB, err)
|
||||
time.Sleep(1747 * time.Millisecond)
|
||||
@@ -251,8 +210,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
|
||||
*syncOptions.concurrency,
|
||||
*syncOptions.aDoDeleteFiles,
|
||||
bFilerSignature,
|
||||
aFilerSignature,
|
||||
&syncStateB2A)
|
||||
aFilerSignature)
|
||||
if err != nil {
|
||||
glog.Errorf("sync from %s to %s: %v", *syncOptions.filerB, *syncOptions.filerA, err)
|
||||
time.Sleep(2147 * time.Millisecond)
|
||||
@@ -283,7 +241,7 @@ func initOffsetFromTsMs(grpcDialOption grpc.DialOption, targetFiler pb.ServerAdd
|
||||
}
|
||||
|
||||
func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceExcludePaths []string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string,
|
||||
replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32, statePtr *atomic.Pointer[syncState]) error {
|
||||
replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool, concurrency int, doDeleteFiles bool, sourceFilerSignature int32, targetFilerSignature int32) error {
|
||||
|
||||
// if first time, start from now
|
||||
// if has previously synced, resume from that point of time
|
||||
@@ -320,17 +278,6 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
|
||||
}
|
||||
processor := NewMetadataProcessor(processEventFn, concurrency, sourceFilerOffsetTsNs)
|
||||
|
||||
// update sync state for graceful shutdown checkpoint saving
|
||||
if statePtr != nil {
|
||||
statePtr.Store(&syncState{
|
||||
processor: processor,
|
||||
grpcDialOption: grpcDialOption,
|
||||
targetFiler: targetFiler,
|
||||
sourcePath: sourcePath,
|
||||
sourceFilerSignature: sourceFilerSignature,
|
||||
})
|
||||
}
|
||||
|
||||
var lastLogTsNs = time.Now().UnixNano()
|
||||
var clientName = fmt.Sprintf("syncFrom_%s_To_%s", string(sourceFiler), string(targetFiler))
|
||||
processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
|
||||
+3
-24
@@ -44,34 +44,13 @@ func init() {
|
||||
|
||||
var cmdIam = &Command{
|
||||
UsageLine: "iam [-port=8111] [-filer=<ip:port>[,<ip:port>]...] [-master=<ip:port>,<ip:port>]",
|
||||
Short: "[DEPRECATED] start a standalone iam API compatible server",
|
||||
Long: `[DEPRECATED] start a standalone iam API compatible server.
|
||||
|
||||
DEPRECATION NOTICE:
|
||||
The standalone 'weed iam' command is deprecated and will be removed in a future release.
|
||||
|
||||
The IAM API is now embedded in the S3 server by default. Simply use 'weed s3' instead,
|
||||
which provides both S3 and IAM APIs on the same port (enabled by default with -iam=true).
|
||||
|
||||
This simplifies deployment by running a single server instead of two separate servers,
|
||||
following the pattern used by MinIO and Ceph RGW.
|
||||
|
||||
To use the embedded IAM API:
|
||||
weed s3 -port=8333 # IAM API is available on the same port
|
||||
|
||||
To disable the embedded IAM API (if you prefer the old behavior):
|
||||
weed s3 -iam=false # Run S3 without IAM
|
||||
weed iam -port=8111 # Run IAM separately (deprecated)
|
||||
Short: "start a iam API compatible server",
|
||||
Long: `start a iam API compatible server.
|
||||
|
||||
Multiple filer addresses can be specified for high availability, separated by commas.`,
|
||||
}
|
||||
|
||||
func runIam(cmd *Command, args []string) bool {
|
||||
glog.Warningf("================================================================================")
|
||||
glog.Warningf("DEPRECATION WARNING: 'weed iam' is deprecated and will be removed in a future release.")
|
||||
glog.Warningf("The IAM API is now embedded in 'weed s3' by default (use -iam=true, which is the default).")
|
||||
glog.Warningf("Please migrate to using 'weed s3' which provides both S3 and IAM APIs on the same port.")
|
||||
glog.Warningf("================================================================================")
|
||||
return iamStandaloneOptions.startIamServer()
|
||||
}
|
||||
|
||||
@@ -110,7 +89,7 @@ func (iamopt *IamOptions) startIamServer() bool {
|
||||
if iamApiServer_err != nil {
|
||||
glog.Fatalf("IAM API Server startup error: %v", iamApiServer_err)
|
||||
}
|
||||
|
||||
|
||||
// Register shutdown handler to prevent goroutine leak
|
||||
grace.OnInterrupt(func() {
|
||||
iamApiServer.Shutdown()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user