Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7e2d4c24de | ||
|
|
202872d88d | ||
|
|
cacfcf5eb2 | ||
|
|
62669040c6 | ||
|
|
777e36bcf3 | ||
|
|
edfe312216 | ||
|
|
489dddf84e | ||
|
|
232d58dc18 | ||
|
|
b2e4474bf2 | ||
|
|
e5fab95905 | ||
|
|
0e8db10a54 | ||
|
|
0c5068cb28 | ||
|
|
d3dabd2d84 | ||
|
|
3e76104d7e | ||
|
|
a770a4983b | ||
|
|
f2634e6a71 | ||
|
|
c1b190cdd6 | ||
|
|
78dda049e5 | ||
|
|
503b0a283c | ||
|
|
eea5d91acb | ||
|
|
e05994ca7e |
@@ -5,6 +5,7 @@
|
||||
# How are we solving the problem?
|
||||
|
||||
|
||||
|
||||
# How is the PR tested?
|
||||
|
||||
|
||||
@@ -12,7 +13,3 @@
|
||||
# Checks
|
||||
- [ ] I have added unit tests if possible.
|
||||
- [ ] I will add related wiki document changes and link to this PR after merging.
|
||||
|
||||
# Checks for AI generated PRs
|
||||
- [ ] I have reviewed every line of code.
|
||||
- [ ] The PR is kept as minimum as possible. Large PRs would not be accepted.
|
||||
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -109,7 +109,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@279495102627de7960cbc33434ab01a12bae144b # v1.22
|
||||
uses: wangyoucao577/go-release-action@ec4e3151b3eccfa3527868ba70f60402265fef06 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -223,23 +223,4 @@ jobs:
|
||||
|
||||
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
|
||||
|
||||
helm-release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: [copy-to-dockerhub]
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
helm_version: "3.18.4"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-e2e-${{ github.sha }}
|
||||
@@ -46,7 +46,6 @@ jobs:
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# Use faster mirrors and install with timeout
|
||||
sudo rm -f /etc/apt/sources.list.d/azure-cli.list /etc/apt/sources.list.d/microsoft-prod.list
|
||||
echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse" | sudo tee /etc/apt/sources.list
|
||||
echo "deb http://azure.archive.ubuntu.com/ubuntu/ $(lsb_release -cs)-updates main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
|
||||
|
||||
@@ -135,7 +134,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: output-logs
|
||||
path: docker/output.log
|
||||
|
||||
@@ -33,29 +33,9 @@ jobs:
|
||||
run: |
|
||||
go test -v
|
||||
|
||||
- name: Collect server logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "Collecting server logs from temp directories..."
|
||||
mkdir -p /tmp/ec-test-logs
|
||||
# Find all temp directories created by the tests (they persist on failure with t.TempDir())
|
||||
find /tmp -maxdepth 1 -type d \( -name "TestEC*" -o -name "TestDisk*" -o -name "TestCross*" -o -name "TestEvacuation*" \) 2>/dev/null | while read dir; do
|
||||
if [ -d "$dir" ]; then
|
||||
echo "Found test directory: $dir"
|
||||
# Copy the entire directory structure to preserve organization
|
||||
cp -r "$dir" /tmp/ec-test-logs/ 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
# List what we collected
|
||||
echo "Collected logs:"
|
||||
find /tmp/ec-test-logs -type f -name "*.log" 2>/dev/null || echo "No logs found"
|
||||
|
||||
- name: Archive logs
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: ec-integration-test-logs
|
||||
path: |
|
||||
/tmp/ec-test-logs/
|
||||
test/erasure_coding/
|
||||
if-no-files-found: warn
|
||||
path: test/erasure_coding
|
||||
@@ -1,49 +0,0 @@
|
||||
name: EC Integration Tests
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/admin/**'
|
||||
- 'weed/worker/**'
|
||||
- 'test/erasure_coding/admin_dockertest/**'
|
||||
- '.github/workflows/ec-integration.yml'
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths:
|
||||
- 'weed/admin/**'
|
||||
- 'weed/worker/**'
|
||||
- 'test/erasure_coding/admin_dockertest/**'
|
||||
- '.github/workflows/ec-integration.yml'
|
||||
|
||||
jobs:
|
||||
ec-integration-test:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.24'
|
||||
|
||||
- name: Build weed binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o ../weed_bin
|
||||
|
||||
- name: Run EC integration tests
|
||||
run: |
|
||||
cd test/erasure_coding/admin_dockertest
|
||||
go test -v -timeout 15m ec_integration_test.go
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ec-test-logs
|
||||
path: test/erasure_coding/admin_dockertest/tmp/logs/
|
||||
retention-days: 7
|
||||
@@ -183,7 +183,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: fuse-integration-test-results
|
||||
path: |
|
||||
|
||||
@@ -33,15 +33,6 @@ jobs:
|
||||
run: |
|
||||
cd weed; go get -v -t -d ./...
|
||||
|
||||
- name: Go Vet (excluding protobuf lock copying)
|
||||
run: |
|
||||
cd weed
|
||||
# Run go vet and filter out known protobuf MessageState lock copying warnings
|
||||
# These are expected in generated protobuf code with embedded sync.Mutex and are safe in practice
|
||||
go vet -v ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-output.txt
|
||||
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
|
||||
if grep -q "vet:" vet-output.txt; then exit 1; fi
|
||||
|
||||
- name: Build
|
||||
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
name: "helm: publish charts"
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@v1.7.0
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
helm_version: "3.18.4"
|
||||
@@ -53,7 +53,6 @@ jobs:
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
|
||||
# Start SeaweedFS with optimized settings for CI
|
||||
# Include S3 API for s3copier integration tests
|
||||
weed server -dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp \
|
||||
-master.electionTimeout=1s \
|
||||
@@ -62,9 +61,6 @@ jobs:
|
||||
-volume.preStopSeconds=1 \
|
||||
-master.peers=none \
|
||||
-filer -filer.maxMB=64 \
|
||||
-s3 -s3.port=8333 \
|
||||
-s3.config="$GITHUB_WORKSPACE/docker/compose/s3.json" \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
-master.port=9333 \
|
||||
-volume.port=8080 \
|
||||
-filer.port=8888 \
|
||||
@@ -109,16 +105,6 @@ jobs:
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Wait for S3 API
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8333/ > /dev/null 2>&1; then
|
||||
echo "✓ S3 API is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 API... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "✓ All SeaweedFS components are ready!"
|
||||
|
||||
# Display cluster status
|
||||
@@ -147,20 +133,6 @@ jobs:
|
||||
run: |
|
||||
mvn test -Dtest=SeaweedFileSystemConfigTest -Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Run S3 ETag Validation Tests (Issue #7768)
|
||||
working-directory: other/java/s3copier
|
||||
env:
|
||||
S3_ENDPOINT: http://127.0.0.1:8333
|
||||
S3_ACCESS_KEY: some_access_key1
|
||||
S3_SECRET_KEY: some_secret_key1
|
||||
run: |
|
||||
echo "Running S3 ETag validation tests against $S3_ENDPOINT"
|
||||
mvn test -Dtest=ETagValidationTest \
|
||||
-DS3_ENDPOINT=$S3_ENDPOINT \
|
||||
-DS3_ACCESS_KEY=$S3_ACCESS_KEY \
|
||||
-DS3_SECRET_KEY=$S3_SECRET_KEY \
|
||||
-Dmaven.javadoc.skip=true -Dgpg.skip=true
|
||||
|
||||
- name: Display logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
|
||||
@@ -49,7 +49,7 @@ jobs:
|
||||
|
||||
- name: Upload Test Reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: test-reports-java-${{ matrix.java }}
|
||||
path: |
|
||||
|
||||
@@ -52,7 +52,6 @@ jobs:
|
||||
run: |
|
||||
cd test/kafka
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
- name: Run Kafka Gateway Unit Tests
|
||||
run: |
|
||||
@@ -97,7 +96,6 @@ jobs:
|
||||
run: |
|
||||
cd test/kafka
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
- name: Run Integration Tests
|
||||
run: |
|
||||
@@ -156,7 +154,6 @@ jobs:
|
||||
cd test/kafka
|
||||
# Use go mod download with timeout to prevent hanging
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
@@ -335,7 +332,6 @@ jobs:
|
||||
cd test/kafka
|
||||
# Use go mod download with timeout to prevent hanging
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
@@ -496,7 +492,6 @@ jobs:
|
||||
run: |
|
||||
cd test/kafka
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
@@ -654,7 +649,6 @@ jobs:
|
||||
run: |
|
||||
cd test/kafka
|
||||
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
|
||||
go mod tidy
|
||||
|
||||
- name: Build and start SeaweedFS MQ
|
||||
run: |
|
||||
@@ -809,7 +803,6 @@ jobs:
|
||||
run: |
|
||||
cd test/kafka
|
||||
go mod download
|
||||
go mod tidy
|
||||
|
||||
- name: Run Protocol Tests
|
||||
run: |
|
||||
|
||||
@@ -70,7 +70,7 @@ jobs:
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: metadata-subscribe-test-logs
|
||||
path: |
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-postgres-${{ github.sha }}
|
||||
@@ -62,7 +62,7 @@ jobs:
|
||||
|
||||
- name: Archive logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: postgres-logs
|
||||
path: test/postgres/postgres-output.log
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
name: "S3 Filer Group Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-filer-group-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
s3-filer-group-tests:
|
||||
name: S3 Filer Group Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Filer Group Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/filer_group
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting Filer Group Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
# The test-with-server target handles server startup/shutdown automatically
|
||||
# Server is started with -filer.group=testgroup
|
||||
make test-with-server || {
|
||||
echo "❌ Filer group tests failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 weed-test.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/filer_group
|
||||
run: |
|
||||
echo "=== Server Logs ==="
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "Last 100 lines of server logs:"
|
||||
tail -100 weed-test.log
|
||||
else
|
||||
echo "No server log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-filer-group-test-logs
|
||||
path: test/s3/filer_group/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-versioning-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -124,7 +124,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-versioning-compatibility-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-cors-compatibility-logs
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -239,7 +239,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-retention-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -306,7 +306,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-cors-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/cors/weed-test*.log
|
||||
@@ -355,7 +355,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-retention-worm-logs
|
||||
path: test/s3/retention/weed-test*.log
|
||||
@@ -365,6 +365,8 @@ jobs:
|
||||
name: S3 Versioning Stress Test
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 35
|
||||
# Only run stress tests on master branch pushes to avoid overloading PR testing
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
@@ -389,9 +391,6 @@ jobs:
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Start server for stress tests
|
||||
make start-server
|
||||
|
||||
# Run stress tests (concurrent operations)
|
||||
make test-versioning-stress || {
|
||||
echo "❌ Stress test failed, checking logs..."
|
||||
@@ -399,30 +398,14 @@ jobs:
|
||||
echo "=== Server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
make stop-server
|
||||
make clean
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Run pagination stress tests (>1000 versions)
|
||||
echo "=== Running pagination stress tests ==="
|
||||
make test-versioning-pagination-stress || {
|
||||
echo "❌ Pagination stress test failed, checking logs..."
|
||||
if [ -f weed-test.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -200 weed-test.log
|
||||
fi
|
||||
make stop-server
|
||||
make clean
|
||||
exit 1
|
||||
}
|
||||
|
||||
make stop-server
|
||||
make clean
|
||||
|
||||
- name: Upload stress test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-versioning-stress-logs
|
||||
path: test/s3/versioning/weed-test*.log
|
||||
@@ -476,65 +459,10 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-tagging-test-logs
|
||||
path: test/s3/tagging/weed-test*.log
|
||||
retention-days: 3
|
||||
|
||||
s3-remote-cache-tests:
|
||||
name: S3 Remote Cache Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Remote Cache Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/remote_cache
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run the remote cache integration tests
|
||||
# Tests singleflight deduplication for caching remote objects
|
||||
make test-with-server || {
|
||||
echo "❌ Test failed, checking logs..."
|
||||
if [ -f primary-weed.log ]; then
|
||||
echo "=== Primary server logs ==="
|
||||
tail -100 primary-weed.log
|
||||
fi
|
||||
if [ -f remote-weed.log ]; then
|
||||
echo "=== Remote server logs ==="
|
||||
tail -100 remote-weed.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-remote-cache-test-logs
|
||||
path: |
|
||||
test/s3/remote_cache/primary-weed.log
|
||||
test/s3/remote_cache/remote-weed.log
|
||||
retention-days: 3
|
||||
|
||||
# Removed SSE-C integration tests and compatibility job
|
||||
@@ -65,7 +65,7 @@ jobs:
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: iam-unit-test-results
|
||||
path: |
|
||||
@@ -162,7 +162,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-iam-integration-logs-${{ matrix.test-type }}
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -222,7 +222,7 @@ jobs:
|
||||
|
||||
- name: Upload distributed test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-iam-distributed-logs
|
||||
path: test/s3/iam/weed-*.log
|
||||
@@ -274,7 +274,7 @@ jobs:
|
||||
|
||||
- name: Upload performance test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-iam-performance-results
|
||||
path: |
|
||||
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-keycloak-test-logs
|
||||
path: |
|
||||
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: test-logs-python-${{ matrix.python-version }}
|
||||
path: |
|
||||
|
||||
@@ -1,391 +0,0 @@
|
||||
name: "S3 Policy Integration Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/s3api/s3_iam_middleware.go'
|
||||
- 'weed/s3api/s3api_bucket_policy*.go'
|
||||
- 'weed/s3api/s3_action_resolver.go'
|
||||
- 'weed/s3api/policy/**'
|
||||
- 'weed/iam/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-policy-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/s3api/s3_iam_middleware.go'
|
||||
- 'weed/s3api/s3api_bucket_policy*.go'
|
||||
- 'weed/s3api/s3_action_resolver.go'
|
||||
- 'weed/s3api/policy/**'
|
||||
- 'weed/iam/**'
|
||||
- 'test/s3/iam/**'
|
||||
- '.github/workflows/s3-policy-tests.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref }}/s3-policy-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
# Unit tests for policy components
|
||||
policy-unit-tests:
|
||||
name: S3 Policy Unit Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
go mod download
|
||||
|
||||
- name: Run S3 Policy Unit Tests
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running S3 Action Resolver Tests ==="
|
||||
go test -v -timeout 5m ./s3api/... -run ".*ActionResolver.*"
|
||||
|
||||
echo "=== Running S3 Bucket Policy Engine Tests ==="
|
||||
go test -v -timeout 5m ./s3api/... -run ".*BucketPolicy.*|.*PolicyEngine.*"
|
||||
|
||||
echo "=== Running IAM Policy Tests ==="
|
||||
go test -v -timeout 5m ./iam/policy/...
|
||||
|
||||
- name: Upload test results on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: policy-unit-test-results
|
||||
path: |
|
||||
weed/testdata/
|
||||
weed/**/testdata/
|
||||
retention-days: 3
|
||||
|
||||
# S3 Policy Variables Integration Tests
|
||||
s3-policy-variables-tests:
|
||||
name: S3 Policy Variables Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 25
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Policy Variables Integration Tests
|
||||
timeout-minutes: 20
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
|
||||
echo "=== Starting S3 Policy Variables Integration Tests ==="
|
||||
|
||||
# Set WEED_BINARY to use the installed version
|
||||
export WEED_BINARY=$(which weed)
|
||||
export TEST_TIMEOUT=15m
|
||||
|
||||
# Run policy variables tests
|
||||
echo "Running policy variables tests..."
|
||||
|
||||
# Kill any existing weed server on port 8333
|
||||
if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
|
||||
kill $(lsof -t -i:8333) 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
# Start weed server with IAM configuration
|
||||
echo "Starting weed server with IAM configuration..."
|
||||
$WEED_BINARY server \
|
||||
-s3 \
|
||||
-s3.port=8333 \
|
||||
-s3.iam.config="$(pwd)/test_iam_config.json" \
|
||||
-filer \
|
||||
-volume.max=0 \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
> /tmp/weed_policy_test_server.log 2>&1 &
|
||||
|
||||
SERVER_PID=$!
|
||||
echo "Server started with PID: $SERVER_PID"
|
||||
|
||||
# Wait for server to be ready
|
||||
echo "Waiting for server to be ready..."
|
||||
MAX_WAIT=30
|
||||
COUNTER=0
|
||||
while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
|
||||
sleep 1
|
||||
COUNTER=$((COUNTER + 1))
|
||||
if [ $COUNTER -ge $MAX_WAIT ]; then
|
||||
echo "Server failed to start within ${MAX_WAIT} seconds"
|
||||
echo "Server log:"
|
||||
cat /tmp/weed_policy_test_server.log
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Server is ready!"
|
||||
|
||||
# Trap to ensure server is killed on exit
|
||||
trap "kill $SERVER_PID 2>/dev/null || true" EXIT
|
||||
|
||||
# Run the tests
|
||||
go test -v -timeout 15m -run TestS3PolicyVariables ./...
|
||||
|
||||
- name: Show service logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
echo "=== Service Logs ==="
|
||||
if [ -f /tmp/weed_policy_test_server.log ]; then
|
||||
echo "--- Last 100 lines of Server Log ---"
|
||||
tail -100 /tmp/weed_policy_test_server.log
|
||||
fi
|
||||
echo ""
|
||||
echo "=== Process Information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-policy-variables-test-logs
|
||||
path: /tmp/weed_policy_test_server.log
|
||||
retention-days: 5
|
||||
|
||||
# S3 Policy Enforcement Integration Tests
|
||||
s3-policy-enforcement-tests:
|
||||
name: S3 Policy Enforcement Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
test-case: ["basic-policy", "contextual-policy", "advanced-policy"]
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Policy Enforcement Tests - ${{ matrix.test-case }}
|
||||
timeout-minutes: 25
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
|
||||
echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ==="
|
||||
|
||||
export WEED_BINARY=$(which weed)
|
||||
export TEST_TIMEOUT=20m
|
||||
|
||||
# Kill any existing weed server on port 8333
|
||||
if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
|
||||
kill $(lsof -t -i:8333) 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
# Start weed server with IAM configuration
|
||||
echo "Starting weed server with IAM configuration..."
|
||||
$WEED_BINARY server \
|
||||
-s3 \
|
||||
-s3.port=8333 \
|
||||
-s3.iam.config="$(pwd)/test_iam_config.json" \
|
||||
-filer \
|
||||
-volume.max=0 \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
> /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log 2>&1 &
|
||||
|
||||
SERVER_PID=$!
|
||||
echo "Server started with PID: $SERVER_PID"
|
||||
|
||||
# Wait for server to be ready
|
||||
echo "Waiting for server to be ready..."
|
||||
MAX_WAIT=30
|
||||
COUNTER=0
|
||||
while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
|
||||
sleep 1
|
||||
COUNTER=$((COUNTER + 1))
|
||||
if [ $COUNTER -ge $MAX_WAIT ]; then
|
||||
echo "Server failed to start within ${MAX_WAIT} seconds"
|
||||
cat /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Server is ready!"
|
||||
|
||||
# Trap to ensure server is killed on exit
|
||||
trap "kill $SERVER_PID 2>/dev/null || true" EXIT
|
||||
|
||||
# Run tests based on test case
|
||||
case "${{ matrix.test-case }}" in
|
||||
"basic-policy")
|
||||
echo "Running basic policy enforcement tests..."
|
||||
go test -v -timeout 20m -run "TestS3IAMBucketPolicy|TestS3IAMPolicyEnforcement" ./...
|
||||
;;
|
||||
"contextual-policy")
|
||||
echo "Running contextual policy tests..."
|
||||
go test -v -timeout 20m -run "TestS3PolicyVariables|TestS3IAMContextual" ./...
|
||||
;;
|
||||
"advanced-policy")
|
||||
echo "Running advanced policy tests..."
|
||||
go test -v -timeout 20m -run "TestS3IAMMultipart|TestS3IAMPresigned" ./...
|
||||
;;
|
||||
*)
|
||||
echo "Unknown test case: ${{ matrix.test-case }}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
- name: Show service logs on failure
|
||||
if: failure()
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
echo "=== Service Logs ==="
|
||||
if [ -f /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log ]; then
|
||||
echo "--- Last 100 lines of Server Log ---"
|
||||
tail -100 /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
|
||||
fi
|
||||
echo ""
|
||||
echo "=== Process Information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp | grep -E "(8333|8888|9333|8080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
|
||||
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
|
||||
retention-days: 5
|
||||
|
||||
# Trusted Proxy Detection Tests
|
||||
trusted-proxy-tests:
|
||||
name: Trusted Proxy Detection Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run Trusted Proxy Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/iam
|
||||
run: |
|
||||
set -x
|
||||
echo "=== Running Trusted Proxy Detection Tests ==="
|
||||
|
||||
export WEED_BINARY=$(which weed)
|
||||
|
||||
# Kill any existing weed server on port 8333
|
||||
if lsof -Pi :8333 -sTCP:LISTEN -t >/dev/null 2>&1 ; then
|
||||
kill $(lsof -t -i:8333) 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
# Start weed server
|
||||
echo "Starting weed server..."
|
||||
$WEED_BINARY server \
|
||||
-s3 \
|
||||
-s3.port=8333 \
|
||||
-s3.iam.config="$(pwd)/test_iam_config.json" \
|
||||
-filer \
|
||||
-volume.max=0 \
|
||||
-master.volumeSizeLimitMB=100 \
|
||||
-s3.allowDeleteBucketNotEmpty=true \
|
||||
> /tmp/weed_proxy_test.log 2>&1 &
|
||||
|
||||
SERVER_PID=$!
|
||||
echo "Server started with PID: $SERVER_PID"
|
||||
|
||||
# Wait for server to be ready
|
||||
echo "Waiting for server to be ready..."
|
||||
MAX_WAIT=30
|
||||
COUNTER=0
|
||||
while ! curl -s http://localhost:8333/status > /dev/null 2>&1; do
|
||||
sleep 1
|
||||
COUNTER=$((COUNTER + 1))
|
||||
if [ $COUNTER -ge $MAX_WAIT ]; then
|
||||
echo "Server failed to start within ${MAX_WAIT} seconds"
|
||||
kill $SERVER_PID 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Trap to ensure server is killed on exit
|
||||
trap "kill $SERVER_PID 2>/dev/null || true" EXIT
|
||||
|
||||
# Run proxy tests
|
||||
go test -v -timeout 10m -run "TestTrustedProxy|TestPrivateIP" ./...
|
||||
|
||||
- name: Show service logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "=== Service Logs ==="
|
||||
if [ -f /tmp/weed_proxy_test.log ]; then
|
||||
echo "--- Last 100 lines of Server Log ---"
|
||||
tail -100 /tmp/weed_proxy_test.log
|
||||
fi
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: trusted-proxy-test-logs
|
||||
path: /tmp/weed_proxy_test.log
|
||||
retention-days: 3
|
||||
@@ -95,7 +95,7 @@ jobs:
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-test-logs-${{ matrix.test-type }}
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -143,7 +143,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-compatibility-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -192,7 +192,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-metadata-persistence-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -241,7 +241,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-copy-operations-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -290,7 +290,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-multipart-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
@@ -340,58 +340,8 @@ jobs:
|
||||
|
||||
- name: Upload performance test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: s3-sse-performance-logs
|
||||
path: test/s3/sse/weed-test*.log
|
||||
retention-days: 7
|
||||
|
||||
s3-volume-encryption:
|
||||
name: S3 Volume Encryption Test
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run S3 Volume Encryption Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/s3/sse
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
|
||||
# Run volume encryption tests with -s3.encryptVolumeData flag
|
||||
echo "🚀 Running S3 volume encryption integration tests..."
|
||||
make test-volume-encryption || {
|
||||
echo "❌ Volume encryption tests failed, checking logs..."
|
||||
if [ -f /tmp/seaweedfs-sse-mini.log ]; then
|
||||
echo "=== Server logs ==="
|
||||
tail -100 /tmp/seaweedfs-sse-mini.log
|
||||
fi
|
||||
echo "=== Process information ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: s3-volume-encryption-logs
|
||||
path: /tmp/seaweedfs-sse-*.log
|
||||
retention-days: 3
|
||||
|
||||
|
||||
@@ -91,4 +91,3 @@ jobs:
|
||||
echo "| readonly | /public | Read-only |" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -125,7 +125,7 @@ jobs:
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: spark-test-results
|
||||
path: test/java/spark/target/surefire-reports/
|
||||
@@ -150,7 +150,7 @@ jobs:
|
||||
- name: Cache Apache Spark
|
||||
if: false && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
|
||||
id: cache-spark
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: spark-3.5.0-bin-hadoop3
|
||||
key: spark-3.5.0-hadoop3
|
||||
|
||||
@@ -18,7 +18,7 @@ defaults:
|
||||
jobs:
|
||||
awscli-tests:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
@@ -103,7 +103,7 @@ jobs:
|
||||
|
||||
- name: Upload server logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
uses: actions/upload-artifact@v5
|
||||
with:
|
||||
name: seaweedfs-logs
|
||||
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
|
||||
|
||||
@@ -1,114 +0,0 @@
|
||||
name: "TUS Protocol Tests"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
- '.github/workflows/tus-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
paths:
|
||||
- 'weed/server/filer_server_tus*.go'
|
||||
- 'weed/server/filer_server.go'
|
||||
- 'test/tus/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.head_ref || github.ref }}/tus-tests
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: weed
|
||||
|
||||
jobs:
|
||||
tus-integration-tests:
|
||||
name: TUS Protocol Integration Tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Install SeaweedFS
|
||||
run: |
|
||||
go install -buildvcs=false
|
||||
|
||||
- name: Run TUS Integration Tests
|
||||
timeout-minutes: 15
|
||||
working-directory: test/tus
|
||||
run: |
|
||||
set -x
|
||||
echo "=== System Information ==="
|
||||
uname -a
|
||||
free -h
|
||||
df -h
|
||||
echo "=== Starting TUS Tests ==="
|
||||
|
||||
# Run tests with automatic server management
|
||||
make test-with-server || {
|
||||
echo "TUS integration tests failed, checking logs..."
|
||||
if [ -f /tmp/seaweedfs-tus-filer.log ]; then
|
||||
echo "=== Filer logs ==="
|
||||
tail -100 /tmp/seaweedfs-tus-filer.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs-tus-master.log ]; then
|
||||
echo "=== Master logs ==="
|
||||
tail -50 /tmp/seaweedfs-tus-master.log
|
||||
fi
|
||||
if [ -f /tmp/seaweedfs-tus-volume.log ]; then
|
||||
echo "=== Volume logs ==="
|
||||
tail -50 /tmp/seaweedfs-tus-volume.log
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Show server logs on failure
|
||||
if: failure()
|
||||
working-directory: test/tus
|
||||
run: |
|
||||
echo "=== Filer Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-filer.log ]; then
|
||||
echo "Last 100 lines of filer logs:"
|
||||
tail -100 /tmp/seaweedfs-tus-filer.log
|
||||
else
|
||||
echo "No filer log file found"
|
||||
fi
|
||||
|
||||
echo "=== Master Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-master.log ]; then
|
||||
tail -50 /tmp/seaweedfs-tus-master.log
|
||||
else
|
||||
echo "No master log file found"
|
||||
fi
|
||||
|
||||
echo "=== Volume Server Logs ==="
|
||||
if [ -f /tmp/seaweedfs-tus-volume.log ]; then
|
||||
tail -50 /tmp/seaweedfs-tus-volume.log
|
||||
else
|
||||
echo "No volume log file found"
|
||||
fi
|
||||
|
||||
echo "=== Test Environment ==="
|
||||
ps aux | grep -E "(weed|test)" || true
|
||||
netstat -tlnp 2>/dev/null | grep -E "(18888|19333|18080)" || true
|
||||
|
||||
- name: Upload test logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: tus-test-logs
|
||||
path: |
|
||||
/tmp/seaweedfs-tus-*.log
|
||||
retention-days: 3
|
||||
@@ -124,20 +124,3 @@ ADVANCED_IAM_DEVELOPMENT_PLAN.md
|
||||
*.log
|
||||
weed-iam
|
||||
test/kafka/kafka-client-loadtest/weed-linux-arm64
|
||||
/test/tus/filerldb2
|
||||
coverage.out
|
||||
/test/s3/remote_cache/test-primary-data
|
||||
/test/s3/remote_cache/test-remote-data
|
||||
test/s3/remote_cache/remote-server.pid
|
||||
test/s3/remote_cache/primary-server.pid
|
||||
/test/erasure_coding/filerldb2
|
||||
/test/s3/cors/test-mini-data
|
||||
/test/s3/filer_group/test-volume-data
|
||||
|
||||
# ID and PID files
|
||||
*.id
|
||||
*.pid
|
||||
test/s3/iam/.test_env
|
||||
/test/erasure_coding/admin_dockertest/tmp
|
||||
/test/erasure_coding/admin_dockertest/task_logs
|
||||
weed_bin
|
||||
|
||||
@@ -54,9 +54,9 @@ Table of Contents
|
||||
=================
|
||||
|
||||
* [Quick Start](#quick-start)
|
||||
* [Quick Start with weed mini](#quick-start-with-weed-mini)
|
||||
* [Quick Start for S3 API on Docker](#quick-start-for-s3-api-on-docker)
|
||||
* [Quick Start with Single Binary](#quick-start-with-single-binary)
|
||||
* [Quick Start SeaweedFS S3 on AWS](#quick-start-seaweedfs-s3-on-aws)
|
||||
* [Introduction](#introduction)
|
||||
* [Features](#features)
|
||||
* [Additional Features](#additional-features)
|
||||
@@ -78,31 +78,6 @@ Table of Contents
|
||||
|
||||
# Quick Start #
|
||||
|
||||
|
||||
## Quick Start with weed mini ##
|
||||
The easiest way to get started with SeaweedFS for development and testing:
|
||||
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
# remove quarantine on macOS
|
||||
# xattr -d com.apple.quarantine ./weed
|
||||
|
||||
./weed mini -dir=/data
|
||||
```
|
||||
|
||||
This single command starts a complete SeaweedFS setup with:
|
||||
- **Master UI**: http://localhost:9333
|
||||
- **Volume Server**: http://localhost:9340
|
||||
- **Filer UI**: http://localhost:8888
|
||||
- **S3 Endpoint**: http://localhost:8333
|
||||
- **WebDAV**: http://localhost:7333
|
||||
- **Admin UI**: http://localhost:23646
|
||||
|
||||
Perfect for development, testing, learning SeaweedFS, and single node deployments!
|
||||
|
||||
## Quick Start for S3 API on Docker ##
|
||||
|
||||
`docker run -p 8333:8333 chrislusf/seaweedfs server -s3`
|
||||
@@ -110,10 +85,13 @@ Perfect for development, testing, learning SeaweedFS, and single node deployment
|
||||
## Quick Start with Single Binary ##
|
||||
* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`. Or run `go install github.com/seaweedfs/seaweedfs/weed@latest`.
|
||||
* `export AWS_ACCESS_KEY_ID=admin ; export AWS_SECRET_ACCESS_KEY=key` as the admin credentials to access the object store.
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway. The difference with `weed mini` is that `weed mini` can auto configure based on the single host environment, while `weed server` requires manual configuration and are designed for production use.
|
||||
* Run `weed server -dir=/some/data/dir -s3` to start one master, one volume server, one filer, and one S3 gateway.
|
||||
|
||||
Also, to increase capacity, just add more volume servers by running `weed volume -dir="/some/data/dir2" -master="<master_host>:9333" -port=8081` locally, or on a different machine, or on thousands of machines. That is it!
|
||||
|
||||
## Quick Start SeaweedFS S3 on AWS ##
|
||||
* Setup fast production-ready [SeaweedFS S3 on AWS with cloudformation](https://aws.amazon.com/marketplace/pp/prodview-nzelz5gprlrjc)
|
||||
|
||||
# Introduction #
|
||||
|
||||
SeaweedFS is a simple and highly scalable distributed file system. There are two objectives:
|
||||
@@ -121,7 +99,7 @@ SeaweedFS is a simple and highly scalable distributed file system. There are two
|
||||
1. to store billions of files!
|
||||
2. to serve the files fast!
|
||||
|
||||
SeaweedFS started as a blob store to handle small files efficiently.
|
||||
SeaweedFS started as an Object Store to handle small files efficiently.
|
||||
Instead of managing all file metadata in a central master,
|
||||
the central master only manages volumes on volume servers,
|
||||
and these volume servers manage files and their metadata.
|
||||
@@ -133,12 +111,16 @@ It is so simple with O(1) disk reads that you are welcome to challenge the perfo
|
||||
|
||||
SeaweedFS started by implementing [Facebook's Haystack design paper](http://www.usenix.org/event/osdi10/tech/full_papers/Beaver.pdf).
|
||||
Also, SeaweedFS implements erasure coding with ideas from
|
||||
[f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf), and has a lot of similarities with [Facebook’s Tectonic Filesystem](https://www.usenix.org/system/files/fast21-pan.pdf) and [Google's Colossus File System](https://cloud.google.com/blog/products/storage-data-transfer/a-peek-behind-colossus-googles-file-system)
|
||||
[f4: Facebook’s Warm BLOB Storage System](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-muralidhar.pdf), and has a lot of similarities with [Facebook’s Tectonic Filesystem](https://www.usenix.org/system/files/fast21-pan.pdf)
|
||||
|
||||
On top of the blob store, optional [Filer] can support directories and POSIX attributes.
|
||||
On top of the object store, optional [Filer] can support directories and POSIX attributes.
|
||||
Filer is a separate linearly-scalable stateless server with customizable metadata stores,
|
||||
e.g., MySql, Postgres, Redis, Cassandra, HBase, Mongodb, Elastic Search, LevelDB, RocksDB, Sqlite, MemSql, TiDB, Etcd, CockroachDB, YDB, etc.
|
||||
|
||||
For any distributed key value stores, the large values can be offloaded to SeaweedFS.
|
||||
With the fast access speed and linearly scalable capacity,
|
||||
SeaweedFS can work as a distributed [Key-Large-Value store][KeyLargeValueStore].
|
||||
|
||||
SeaweedFS can transparently integrate with the cloud.
|
||||
With hot data on local cluster, and warm data on the cloud with O(1) access time,
|
||||
SeaweedFS can achieve both fast local access time and elastic cloud storage capacity.
|
||||
@@ -148,13 +130,13 @@ Faster and cheaper than direct cloud storage!
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
# Features #
|
||||
## Additional Blob Store Features ##
|
||||
* Support different replication levels, with rack and data center aware.
|
||||
## Additional Features ##
|
||||
* Can choose no replication or different replication levels, rack and data center aware.
|
||||
* Automatic master servers failover - no single point of failure (SPOF).
|
||||
* Automatic compression depending on file MIME type.
|
||||
* Automatic Gzip compression depending on file MIME type.
|
||||
* Automatic compaction to reclaim disk space after deletion or update.
|
||||
* [Automatic entry TTL expiration][VolumeServerTTL].
|
||||
* Flexible Capacity Expansion: Any server with some disk space can add to the total storage space.
|
||||
* Any server with some disk space can add to the total storage space.
|
||||
* Adding/Removing servers does **not** cause any data re-balancing unless triggered by admin commands.
|
||||
* Optional picture resizing.
|
||||
* Support ETag, Accept-Range, Last-Modified, etc.
|
||||
@@ -162,7 +144,7 @@ Faster and cheaper than direct cloud storage!
|
||||
* Support rebalancing the writable and readonly volumes.
|
||||
* [Customizable Multiple Storage Tiers][TieredStorage]: Customizable storage disk types to balance performance and cost.
|
||||
* [Transparent cloud integration][CloudTier]: unlimited capacity via tiered cloud storage for warm data.
|
||||
* [Erasure Coding for warm storage][ErasureCoding] Rack-Aware 10.4 erasure coding reduces storage cost and increases availability. Enterprise version can customize EC ratio.
|
||||
* [Erasure Coding for warm storage][ErasureCoding] Rack-Aware 10.4 erasure coding reduces storage cost and increases availability.
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
@@ -208,7 +190,7 @@ Faster and cheaper than direct cloud storage!
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
## Example: Using Seaweed Blob Store ##
|
||||
## Example: Using Seaweed Object Store ##
|
||||
|
||||
By default, the master node runs on port 9333, and the volume nodes run on port 8080.
|
||||
Let's start one master node, and two volume nodes on port 8080 and 8081. Ideally, they should be started from different machines. We'll use localhost as an example.
|
||||
@@ -228,25 +210,23 @@ SeaweedFS uses HTTP REST operations to read, write, and delete. The responses ar
|
||||
> weed volume -dir="/tmp/data2" -max=10 -master="localhost:9333" -port=8081 &
|
||||
```
|
||||
|
||||
### Write A Blob ###
|
||||
### Write File ###
|
||||
|
||||
A blob, also referred as a needle, a chunk, or mistakenly as a file, is just a byte array. It can have attributes, such as name, mime type, create or update time, etc. But basically it is just a byte array of a relatively small size, such as 2 MB ~ 64 MB. The size is not fixed.
|
||||
|
||||
To upload a blob: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server URL:
|
||||
To upload a file: first, send a HTTP POST, PUT, or GET request to `/dir/assign` to get an `fid` and a volume server URL:
|
||||
|
||||
```
|
||||
> curl http://localhost:9333/dir/assign
|
||||
{"count":1,"fid":"3,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"}
|
||||
```
|
||||
|
||||
Second, to store the blob content, send a HTTP multi-part POST request to `url + '/' + fid` from the response:
|
||||
Second, to store the file content, send a HTTP multi-part POST request to `url + '/' + fid` from the response:
|
||||
|
||||
```
|
||||
> curl -F file=@/home/chris/myphoto.jpg http://127.0.0.1:8080/3,01637037d6
|
||||
{"name":"myphoto.jpg","size":43234,"eTag":"1cc0118e"}
|
||||
```
|
||||
|
||||
To update, send another POST request with updated blob content.
|
||||
To update, send another POST request with updated file content.
|
||||
|
||||
For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
|
||||
|
||||
@@ -254,7 +234,7 @@ For deletion, send an HTTP DELETE request to the same `url + '/' + fid` URL:
|
||||
> curl -X DELETE http://127.0.0.1:8080/3,01637037d6
|
||||
```
|
||||
|
||||
### Save Blob Id ###
|
||||
### Save File Id ###
|
||||
|
||||
Now, you can save the `fid`, 3,01637037d6 in this case, to a database field.
|
||||
|
||||
@@ -266,9 +246,9 @@ The file key and file cookie are both coded in hex. You can store the <volume id
|
||||
|
||||
If stored as a string, in theory, you would need 8+1+16+8=33 bytes. A char(33) would be enough, if not more than enough, since most uses will not need 2^32 volumes.
|
||||
|
||||
If space is really a concern, you can store the file id in the binary format. You would need one 4-byte integer for volume id, 8-byte long number for file key, and a 4-byte integer for the file cookie. So 16 bytes are more than enough.
|
||||
If space is really a concern, you can store the file id in your own format. You would need one 4-byte integer for volume id, 8-byte long number for file key, and a 4-byte integer for the file cookie. So 16 bytes are more than enough.
|
||||
|
||||
### Read a Blob ###
|
||||
### Read File ###
|
||||
|
||||
Here is an example of how to render the URL.
|
||||
|
||||
@@ -309,7 +289,7 @@ http://localhost:8080/3/01637037d6.jpg?height=200&width=200&mode=fill
|
||||
|
||||
### Rack-Aware and Data Center-Aware Replication ###
|
||||
|
||||
SeaweedFS applies the replication strategy at a volume level. So, when you are getting a blob id, you can specify the replication strategy. For example:
|
||||
SeaweedFS applies the replication strategy at a volume level. So, when you are getting a file id, you can specify the replication strategy. For example:
|
||||
|
||||
```
|
||||
curl http://localhost:9333/dir/assign?replication=001
|
||||
@@ -332,7 +312,7 @@ More details about replication can be found [on the wiki][Replication].
|
||||
|
||||
You can also set the default replication strategy when starting the master server.
|
||||
|
||||
### Allocate Blob Key on Specific Data Center ###
|
||||
### Allocate File Key on Specific Data Center ###
|
||||
|
||||
Volume servers can be started with a specific data center name:
|
||||
|
||||
@@ -341,7 +321,7 @@ Volume servers can be started with a specific data center name:
|
||||
weed volume -dir=/tmp/2 -port=8081 -dataCenter=dc2
|
||||
```
|
||||
|
||||
When requesting a blob key, an optional "dataCenter" parameter can limit the assigned volume to the specific data center. For example, this specifies that the assigned volume should be limited to 'dc1':
|
||||
When requesting a file key, an optional "dataCenter" parameter can limit the assigned volume to the specific data center. For example, this specifies that the assigned volume should be limited to 'dc1':
|
||||
|
||||
```
|
||||
http://localhost:9333/dir/assign?dataCenter=dc1
|
||||
@@ -360,15 +340,15 @@ When requesting a blob key, an optional "dataCenter" parameter can limit the ass
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
## Blob Store Architecture ##
|
||||
## Object Store Architecture ##
|
||||
|
||||
Usually distributed file systems split each file into chunks. A central server keeps a mapping of filenames to chunks, and also which chunks each chunk server has.
|
||||
Usually distributed file systems split each file into chunks, a central master keeps a mapping of filenames, chunk indices to chunk handles, and also which chunks each chunk server has.
|
||||
|
||||
The main drawback is that the central server can't handle many small files efficiently, and since all read requests need to go through the central master, so it might not scale well for many concurrent users.
|
||||
The main drawback is that the central master can't handle many small files efficiently, and since all read requests need to go through the chunk master, so it might not scale well for many concurrent users.
|
||||
|
||||
Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is 32GB in size, and can hold a lot of blobs. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is a fairly small amount of data and is generally stable.
|
||||
Instead of managing chunks, SeaweedFS manages data volumes in the master server. Each data volume is 32GB in size, and can hold a lot of files. And each storage node can have many data volumes. So the master node only needs to store the metadata about the volumes, which is a fairly small amount of data and is generally stable.
|
||||
|
||||
The actual blob metadata, which are the blob volume, offset, and size, is stored in each volume on volume servers. Since each volume server only manages metadata of blobs on its own disk, with only 16 bytes for each blob, all access can read the metadata just from memory and only needs one disk operation to actually read file data.
|
||||
The actual file metadata is stored in each volume on volume servers. Since each volume server only manages metadata of files on its own disk, with only 16 bytes for each file, all file access can read file metadata just from memory and only needs one disk operation to actually read file data.
|
||||
|
||||
For comparison, consider that an xfs inode structure in Linux is 536 bytes.
|
||||
|
||||
@@ -382,13 +362,23 @@ On each write request, the master server also generates a file key, which is a g
|
||||
|
||||
### Write and Read files ###
|
||||
|
||||
When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node URL) for the blob. The client then contacts the volume node and POSTs the blob content.
|
||||
When a client sends a write request, the master server returns (volume id, file key, file cookie, volume node URL) for the file. The client then contacts the volume node and POSTs the file content.
|
||||
|
||||
When a client needs to read a blob based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node URL, volume node public URL), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
|
||||
When a client needs to read a file based on (volume id, file key, file cookie), it asks the master server by the volume id for the (volume node URL, volume node public URL), or retrieves this from a cache. Then the client can GET the content, or just render the URL on web pages and let browsers fetch the content.
|
||||
|
||||
Please see the example for details on the write-read process.
|
||||
|
||||
### Storage Size ###
|
||||
|
||||
In the current implementation, each volume can hold 32 gibibytes (32GiB or 8x2^32 bytes). This is because we align content to 8 bytes. We can easily increase this to 64GiB, or 128GiB, or more, by changing 2 lines of code, at the cost of some wasted padding space due to alignment.
|
||||
|
||||
There can be 4 gibibytes (4GiB or 2^32 bytes) of volumes. So the total system size is 8 x 4GiB x 4GiB which is 128 exbibytes (128EiB or 2^67 bytes).
|
||||
|
||||
Each individual file size is limited to the volume size.
|
||||
|
||||
### Saving memory ###
|
||||
|
||||
All blob metadata stored on a volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
|
||||
All file meta information stored on a volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
|
||||
|
||||
### Tiered Storage to the cloud ###
|
||||
|
||||
@@ -402,12 +392,6 @@ If the hot/warm data is split as 20/80, with 20 servers, you can achieve storage
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
## SeaweedFS Filer ##
|
||||
|
||||
Built on top of the blob store, SeaweedFS Filer adds directory structure to create a file system. The directory sturcture is an interface that is implemented in many key-value stores or databases.
|
||||
|
||||
The content of a file is mapped to one or many blobs, distributed to multiple volumes on multiple volume servers.
|
||||
|
||||
## Compared to Other File Systems ##
|
||||
|
||||
Most other distributed file systems seem more complicated than necessary.
|
||||
@@ -654,4 +638,5 @@ The text of this page is available for modification and reuse under the terms of
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
## Stargazers over time
|
||||
[](https://starchart.cc/seaweedfs/seaweedfs)
|
||||
|
||||
[](https://starchart.cc/chrislusf/seaweedfs)
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
# Server-Side Encryption with Customer-Provided Keys (SSE-C) Implementation
|
||||
|
||||
This document describes the implementation of SSE-C support in SeaweedFS, addressing the feature request from [GitHub Discussion #5361](https://github.com/seaweedfs/seaweedfs/discussions/5361).
|
||||
|
||||
## Overview
|
||||
|
||||
SSE-C allows clients to provide their own encryption keys for server-side encryption of objects stored in SeaweedFS. The server encrypts the data using the customer-provided AES-256 key but does not store the key itself - only an MD5 hash of the key for validation purposes.
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Architecture
|
||||
|
||||
The SSE-C implementation follows a transparent encryption/decryption pattern:
|
||||
|
||||
1. **Upload (PUT/POST)**: Data is encrypted with the customer key before being stored
|
||||
2. **Download (GET/HEAD)**: Encrypted data is decrypted on-the-fly using the customer key
|
||||
3. **Metadata Storage**: Only the encryption algorithm and key MD5 are stored as metadata
|
||||
|
||||
### Key Components
|
||||
|
||||
#### 1. Constants and Headers (`weed/s3api/s3_constants/header.go`)
|
||||
- Added AWS-compatible SSE-C header constants
|
||||
- Support for both regular and copy-source SSE-C headers
|
||||
|
||||
#### 2. Core SSE-C Logic (`weed/s3api/s3_sse_c.go`)
|
||||
- **SSECustomerKey**: Structure to hold customer encryption key and metadata
|
||||
- **SSECEncryptedReader**: Streaming encryption with AES-256-CTR mode
|
||||
- **SSECDecryptedReader**: Streaming decryption with IV extraction
|
||||
- **validateAndParseSSECHeaders**: Shared validation logic (DRY principle)
|
||||
- **ParseSSECHeaders**: Parse regular SSE-C headers
|
||||
- **ParseSSECCopySourceHeaders**: Parse copy-source SSE-C headers
|
||||
- Header validation and parsing functions
|
||||
- Metadata extraction and response handling
|
||||
|
||||
#### 3. Error Handling (`weed/s3api/s3err/s3api_errors.go`)
|
||||
- New error codes for SSE-C validation failures
|
||||
- AWS-compatible error messages and HTTP status codes
|
||||
|
||||
#### 4. S3 API Integration
|
||||
- **PUT Object Handler**: Encrypts data streams transparently
|
||||
- **GET Object Handler**: Decrypts data streams transparently
|
||||
- **HEAD Object Handler**: Validates keys and returns appropriate headers
|
||||
- **Metadata Storage**: Integrates with existing `SaveAmzMetaData` function
|
||||
|
||||
### Encryption Scheme
|
||||
|
||||
- **Algorithm**: AES-256-CTR (Counter mode)
|
||||
- **Key Size**: 256 bits (32 bytes)
|
||||
- **IV Generation**: Random 16-byte IV per object
|
||||
- **Storage Format**: `[IV][EncryptedData]` where IV is prepended to encrypted content
|
||||
|
||||
### Metadata Storage
|
||||
|
||||
SSE-C metadata is stored in the filer's extended attributes:
|
||||
```
|
||||
x-amz-server-side-encryption-customer-algorithm: "AES256"
|
||||
x-amz-server-side-encryption-customer-key-md5: "<md5-hash-of-key>"
|
||||
```
|
||||
|
||||
## API Compatibility
|
||||
|
||||
### Required Headers for Encryption (PUT/POST)
|
||||
```
|
||||
x-amz-server-side-encryption-customer-algorithm: AES256
|
||||
x-amz-server-side-encryption-customer-key: <base64-encoded-256-bit-key>
|
||||
x-amz-server-side-encryption-customer-key-md5: <md5-hash-of-key>
|
||||
```
|
||||
|
||||
### Required Headers for Decryption (GET/HEAD)
|
||||
Same headers as encryption - the server validates the key MD5 matches.
|
||||
|
||||
### Copy Operations
|
||||
Support for copy-source SSE-C headers:
|
||||
```
|
||||
x-amz-copy-source-server-side-encryption-customer-algorithm
|
||||
x-amz-copy-source-server-side-encryption-customer-key
|
||||
x-amz-copy-source-server-side-encryption-customer-key-md5
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
The implementation provides AWS-compatible error responses:
|
||||
|
||||
- **InvalidEncryptionAlgorithmError**: Non-AES256 algorithm specified
|
||||
- **InvalidArgument**: Invalid key format, size, or MD5 mismatch
|
||||
- **Missing customer key**: Object encrypted but no key provided
|
||||
- **Unnecessary customer key**: Object not encrypted but key provided
|
||||
|
||||
## Security Considerations
|
||||
|
||||
1. **Key Management**: Customer keys are never stored - only MD5 hashes for validation
|
||||
2. **IV Randomness**: Fresh random IV generated for each object
|
||||
3. **Transparent Security**: Volume servers never see unencrypted data
|
||||
4. **Key Validation**: Strict validation of key format, size, and MD5
|
||||
|
||||
## Testing
|
||||
|
||||
Comprehensive test suite covers:
|
||||
- Header validation and parsing (regular and copy-source)
|
||||
- Encryption/decryption round-trip
|
||||
- Error condition handling
|
||||
- Metadata extraction
|
||||
- Code reuse validation (DRY principle)
|
||||
- AWS S3 compatibility
|
||||
|
||||
Run tests with:
|
||||
```bash
|
||||
go test -v ./weed/s3api
|
||||
|
||||
## Usage Example
|
||||
|
||||
### Upload with SSE-C
|
||||
```bash
|
||||
# Generate a 256-bit key
|
||||
KEY=$(openssl rand -base64 32)
|
||||
KEY_MD5=$(echo -n "$KEY" | base64 -d | openssl dgst -md5 -binary | base64)
|
||||
|
||||
# Upload object with SSE-C
|
||||
curl -X PUT "http://localhost:8333/bucket/object" \
|
||||
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
|
||||
-H "x-amz-server-side-encryption-customer-key: $KEY" \
|
||||
-H "x-amz-server-side-encryption-customer-key-md5: $KEY_MD5" \
|
||||
--data-binary @file.txt
|
||||
```
|
||||
|
||||
### Download with SSE-C
|
||||
```bash
|
||||
# Download object with SSE-C (same key required)
|
||||
curl "http://localhost:8333/bucket/object" \
|
||||
-H "x-amz-server-side-encryption-customer-algorithm: AES256" \
|
||||
-H "x-amz-server-side-encryption-customer-key: $KEY" \
|
||||
-H "x-amz-server-side-encryption-customer-key-md5: $KEY_MD5"
|
||||
```
|
||||
|
||||
## Integration Points
|
||||
|
||||
### Existing SeaweedFS Features
|
||||
- **Filer Metadata**: Extends existing metadata storage
|
||||
- **Volume Servers**: No changes required - store encrypted data transparently
|
||||
- **S3 API**: Integrates seamlessly with existing handlers
|
||||
- **Versioning**: Compatible with object versioning
|
||||
- **Multipart Upload**: Ready for multipart upload integration
|
||||
|
||||
### Future Enhancements
|
||||
- **SSE-S3**: Server-managed encryption keys
|
||||
- **SSE-KMS**: External key management service integration
|
||||
- **Performance Optimization**: Hardware acceleration for encryption
|
||||
- **Compliance**: Enhanced audit logging for encrypted objects
|
||||
|
||||
## File Changes Summary
|
||||
|
||||
1. **`weed/s3api/s3_constants/header.go`** - Added SSE-C header constants
|
||||
2. **`weed/s3api/s3_sse_c.go`** - Core SSE-C implementation (NEW)
|
||||
3. **`weed/s3api/s3_sse_c_test.go`** - Comprehensive test suite (NEW)
|
||||
4. **`weed/s3api/s3err/s3api_errors.go`** - Added SSE-C error codes
|
||||
5. **`weed/s3api/s3api_object_handlers.go`** - GET/HEAD with SSE-C support
|
||||
6. **`weed/s3api/s3api_object_handlers_put.go`** - PUT with SSE-C support
|
||||
7. **`weed/server/filer_server_handlers_write_autochunk.go`** - Metadata storage
|
||||
|
||||
## Compliance
|
||||
|
||||
This implementation follows the [AWS S3 SSE-C specification](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html) for maximum compatibility with existing S3 clients and tools.
|
||||
|
||||
## Performance Impact
|
||||
|
||||
- **Encryption Overhead**: Minimal CPU impact with efficient AES-CTR streaming
|
||||
- **Memory Usage**: Constant memory usage via streaming encryption/decryption
|
||||
- **Storage Overhead**: 16 bytes per object for IV storage
|
||||
- **Network**: No additional network overhead
|
||||
@@ -1,11 +1,13 @@
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.24.9
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.1
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go v0.121.6 // indirect
|
||||
cloud.google.com/go/pubsub v1.50.1
|
||||
cloud.google.com/go/storage v1.59.1
|
||||
cloud.google.com/go/storage v1.57.1
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.55.8
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
@@ -13,7 +15,7 @@ require (
|
||||
github.com/cenkalti/backoff/v4 v4.3.0
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
@@ -26,16 +28,18 @@ require (
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-redsync/redsync/v4 v4.15.0
|
||||
github.com/go-redsync/redsync/v4 v4.14.0
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/gocql/gocql v1.7.0
|
||||
github.com/golang/protobuf v1.5.4
|
||||
github.com/golang/snappy v1.0.0
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/google/wire v0.7.0 // indirect
|
||||
github.com/google/wire v0.6.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
@@ -47,10 +51,10 @@ require (
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.2
|
||||
github.com/klauspost/reedsolomon v1.13.0
|
||||
github.com/klauspost/reedsolomon v1.12.6
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.3
|
||||
github.com/mailru/easyjson v0.9.1 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -62,13 +66,13 @@ require (
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.19.2
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.6
|
||||
github.com/sirupsen/logrus v1.9.4-0.20230606125235-dd1b4c2e81af // indirect
|
||||
github.com/seaweedfs/raft v1.1.3
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/viper v1.21.0
|
||||
@@ -89,27 +93,27 @@ require (
|
||||
go.etcd.io/etcd/client/v3 v3.6.6
|
||||
go.mongodb.org/mongo-driver v1.17.6
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.44.0
|
||||
gocloud.dev/pubsub/natspubsub v0.44.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.44.0
|
||||
golang.org/x/crypto v0.46.0
|
||||
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
|
||||
golang.org/x/image v0.35.0
|
||||
golang.org/x/net v0.48.0
|
||||
golang.org/x/oauth2 v0.34.0
|
||||
golang.org/x/sys v0.39.0
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
golang.org/x/tools v0.40.0 // indirect
|
||||
gocloud.dev v0.43.0
|
||||
gocloud.dev/pubsub/natspubsub v0.43.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.43.0
|
||||
golang.org/x/crypto v0.45.0
|
||||
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
|
||||
golang.org/x/image v0.33.0
|
||||
golang.org/x/net v0.47.0
|
||||
golang.org/x/oauth2 v0.32.0 // indirect
|
||||
golang.org/x/sys v0.38.0
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/tools v0.38.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
google.golang.org/api v0.258.0
|
||||
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
|
||||
google.golang.org/grpc v1.78.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
google.golang.org/api v0.247.0
|
||||
google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect
|
||||
google.golang.org/grpc v1.77.0
|
||||
google.golang.org/protobuf v1.36.10
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/mathutil v1.7.1
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
modernc.org/sqlite v1.44.2
|
||||
modernc.org/sqlite v1.39.0
|
||||
modernc.org/strutil v1.2.1
|
||||
)
|
||||
|
||||
@@ -119,46 +123,44 @@ require (
|
||||
github.com/Jille/raft-grpc-transport v1.6.1
|
||||
github.com/ThreeDotsLabs/watermill v1.5.1
|
||||
github.com/a-h/templ v0.3.943
|
||||
github.com/apache/cassandra-gocql-driver/v2 v2.0.0
|
||||
github.com/apple/foundationdb/bindings/go v0.0.0-20250911184653-27f7192f47c3
|
||||
github.com/arangodb/go-driver v1.6.9
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.7
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.7
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.40.0
|
||||
github.com/getsentry/sentry-go v0.38.0
|
||||
github.com/gin-contrib/sessions v1.0.4
|
||||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.12
|
||||
github.com/golang-jwt/jwt/v5 v5.3.0
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hanwen/go-fuse/v2 v2.9.0
|
||||
github.com/hashicorp/raft v1.7.3
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.3.1
|
||||
github.com/hashicorp/vault/api v1.22.0
|
||||
github.com/jhump/protoreflect v1.17.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/linkedin/goavro/v2 v2.14.1
|
||||
github.com/mattn/go-sqlite3 v1.14.33
|
||||
github.com/mattn/go-sqlite3 v1.14.32
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.26.4
|
||||
github.com/parquet-go/parquet-go v0.25.1
|
||||
github.com/pkg/sftp v1.13.10
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rclone/rclone v1.72.1
|
||||
github.com/rclone/rclone v1.71.2
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1
|
||||
github.com/shirou/gopsutil/v4 v4.25.12
|
||||
github.com/redis/go-redis/v9 v9.14.1
|
||||
github.com/schollz/progressbar/v3 v3.18.0
|
||||
github.com/shirou/gopsutil/v4 v4.25.11
|
||||
github.com/tarantool/go-tarantool/v2 v2.4.1
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.125.1
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.7
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.121.0
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.6.6
|
||||
go.uber.org/atomic v1.11.0
|
||||
golang.org/x/sync v0.19.0
|
||||
golang.org/x/tools/godoc v0.1.0-deprecated
|
||||
@@ -168,35 +170,23 @@ require (
|
||||
require github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
|
||||
|
||||
require (
|
||||
cloud.google.com/go/longrunning v0.7.0 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
|
||||
cloud.google.com/go/longrunning v0.6.7 // indirect
|
||||
cloud.google.com/go/pubsub/v2 v2.0.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
|
||||
github.com/a1ex3/zstd-seekable-format-go/pkg v0.10.0 // indirect
|
||||
github.com/anchore/go-lzo v0.1.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/bazelbuild/rules_go v0.46.0 // indirect
|
||||
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
|
||||
github.com/blevesearch/snowballstem v0.9.0 // indirect
|
||||
github.com/boombuler/barcode v1.1.0 // indirect
|
||||
github.com/bufbuild/protocompile v0.14.1 // indirect
|
||||
github.com/buger/jsonparser v1.1.1 // indirect
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
|
||||
github.com/cockroachdb/apd/v3 v3.1.0 // indirect
|
||||
github.com/cockroachdb/errors v1.11.3 // indirect
|
||||
github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 // indirect
|
||||
github.com/cockroachdb/redact v1.1.5 // indirect
|
||||
github.com/cockroachdb/version v0.0.0-20250314144055-3860cd14adf2 // indirect
|
||||
github.com/dave/dst v0.27.2 // indirect
|
||||
github.com/diskfs/go-diskfs v1.7.0 // indirect
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/gopherjs/gopherjs v1.17.2 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
|
||||
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
|
||||
github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
|
||||
@@ -207,27 +197,20 @@ require (
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/jaegertracing/jaeger v1.47.0 // indirect
|
||||
github.com/jtolds/gls v4.20.0+incompatible // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
|
||||
github.com/openzipkin/zipkin-go v0.4.3 // indirect
|
||||
github.com/parquet-go/bitpack v1.0.0 // indirect
|
||||
github.com/parquet-go/jsonlite v1.0.0 // indirect
|
||||
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
|
||||
github.com/pierrre/geohash v1.0.0 // indirect
|
||||
github.com/pquerna/otp v1.5.0 // indirect
|
||||
github.com/quic-go/qpack v0.6.0 // indirect
|
||||
github.com/quic-go/quic-go v0.57.0 // indirect
|
||||
github.com/quic-go/qpack v0.5.1 // indirect
|
||||
github.com/quic-go/quic-go v0.54.1 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/ryanuber/go-glob v1.0.0 // indirect
|
||||
github.com/sasha-s/go-deadlock v0.3.1 // indirect
|
||||
github.com/smarty/assertions v1.15.0 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/twpayne/go-geom v1.4.1 // indirect
|
||||
github.com/twpayne/go-kml v1.5.2 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||
@@ -235,31 +218,32 @@ require (
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
|
||||
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.uber.org/mock v0.5.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.31.0 // indirect
|
||||
golang.org/x/mod v0.29.0 // indirect
|
||||
gonum.org/v1/gonum v0.16.0 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.24.0 // indirect
|
||||
cloud.google.com/go/auth v0.17.0 // indirect
|
||||
cloud.google.com/go/auth v0.16.5 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
cloud.google.com/go/iam v1.5.2 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.2 // indirect
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.2-0.20251110135918-10b7b7e7cd26 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.2 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.264 // indirect
|
||||
github.com/Files-com/files-sdk-go/v3 v3.2.218 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.54.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect
|
||||
github.com/IBM/go-sdk-core/v5 v5.21.0 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.2 // indirect
|
||||
@@ -275,22 +259,23 @@ require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
@@ -301,7 +286,7 @@ require (
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/chilts/sid v0.0.0-20190607042430-660e94789ec9 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
|
||||
github.com/cloudinary/cloudinary-go/v2 v2.12.0 // indirect
|
||||
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
|
||||
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
|
||||
github.com/cloudwego/base64x v0.1.6 // indirect
|
||||
@@ -322,28 +307,28 @@ require (
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/flynn/noise v1.1.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.11 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.9 // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
github.com/gin-contrib/sse v1.1.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.3 // indirect
|
||||
github.com/go-chi/chi/v5 v5.2.2 // indirect
|
||||
github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.4 // indirect
|
||||
github.com/go-openapi/strfmt v0.25.0 // indirect
|
||||
github.com/go-openapi/errors v0.22.2 // indirect
|
||||
github.com/go-openapi/strfmt v0.23.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.28.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.27.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.16.5 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/gofrs/flock v0.13.0 // indirect
|
||||
github.com/gofrs/flock v0.12.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
|
||||
github.com/gorilla/context v1.1.2 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||
@@ -374,27 +359,27 @@ require (
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/lanrat/extsort v1.4.2 // indirect
|
||||
github.com/lanrat/extsort v1.4.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lpar/date v1.0.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.16 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
|
||||
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/nats-io/nats.go v1.43.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.11 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.5 // indirect
|
||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.4 // indirect
|
||||
github.com/nxadm/tail v1.4.11 // indirect
|
||||
github.com/oklog/ulid v1.3.1 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.104.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.98.0 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.3 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
@@ -410,25 +395,26 @@ require (
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
github.com/relvacode/iso8601 v1.7.0 // indirect
|
||||
github.com/relvacode/iso8601 v1.6.0 // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/samber/lo v1.52.0 // indirect
|
||||
github.com/samber/lo v1.51.0 // indirect
|
||||
github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
|
||||
github.com/smartystreets/goconvey v1.8.1 // indirect
|
||||
github.com/sony/gobreaker v1.0.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.25-0.20251022131615-eb24eb109368 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.24 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20251031123324-a804aaa87491 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20250926104142-ccb8d3498e6c // indirect
|
||||
github.com/tarantool/go-iproto v1.1.0 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tinylib/msgp v1.5.0 // indirect
|
||||
github.com/tinylib/msgp v1.3.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.16 // indirect
|
||||
github.com/tklauser/numcpus v0.11.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
@@ -446,12 +432,12 @@ require (
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
github.com/zeebo/blake3 v0.2.4 // indirect
|
||||
github.com/zeebo/errs v1.4.0 // indirect
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
go.etcd.io/bbolt v1.4.2 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.6.6 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.62.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
|
||||
go.opentelemetry.io/otel v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.38.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
|
||||
@@ -460,18 +446,18 @@ require (
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.0 // indirect
|
||||
golang.org/x/arch v0.20.0 // indirect
|
||||
golang.org/x/term v0.38.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
|
||||
golang.org/x/term v0.37.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.67.6 // indirect
|
||||
modernc.org/libc v1.66.3 // indirect
|
||||
moul.io/http2curl/v2 v2.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
storj.io/common v0.0.0-20251107171817-6221ae45072c // indirect
|
||||
storj.io/common v0.0.0-20250808122759-804533d519c1 // indirect
|
||||
storj.io/drpc v0.0.35-0.20250513201419-f7819ea69b55 // indirect
|
||||
storj.io/eventkit v0.0.0-20250410172343-61f26d3de156 // indirect
|
||||
storj.io/infectious v0.0.2 // indirect
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.07"
|
||||
appVersion: "4.02"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.0.407
|
||||
version: 4.0.402
|
||||
|
||||
@@ -1595,101 +1595,6 @@
|
||||
"title": "S3 Bucket Traffic Sent",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "reqps",
|
||||
"unitScale": true
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 41
|
||||
},
|
||||
"id": 86,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$NAMESPACE\"}[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "S3 API Calls per Bucket",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
@@ -1754,8 +1659,8 @@
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 41
|
||||
},
|
||||
"id": 72,
|
||||
@@ -3361,209 +3266,6 @@
|
||||
],
|
||||
"title": "Filer Go Routines",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 48
|
||||
},
|
||||
"id": 89,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}} (logical)",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}} (physical)",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "S3 Bucket Size",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 48
|
||||
},
|
||||
"id": 90,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "S3 Bucket Object Count",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "",
|
||||
@@ -3654,4 +3356,4 @@
|
||||
"uid": "a24009d7-cbda-4443-a132-1cc1c4677304",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,12 @@ spec:
|
||||
port: {{ .Values.admin.grpcPort }}
|
||||
targetPort: {{ .Values.admin.grpcPort }}
|
||||
protocol: TCP
|
||||
{{- if .Values.admin.metricsPort }}
|
||||
- name: "metrics"
|
||||
port: {{ .Values.admin.metricsPort }}
|
||||
targetPort: {{ .Values.admin.metricsPort }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
{{- if .Values.admin.enabled }}
|
||||
{{- if .Values.admin.metricsPort }}
|
||||
{{- if .Values.global.monitoring.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
@@ -21,7 +22,7 @@ metadata:
|
||||
spec:
|
||||
endpoints:
|
||||
- interval: 30s
|
||||
port: http
|
||||
port: metrics
|
||||
scrapeTimeout: 5s
|
||||
selector:
|
||||
matchLabels:
|
||||
@@ -29,3 +30,4 @@ spec:
|
||||
app.kubernetes.io/component: admin
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -119,7 +119,7 @@ spec:
|
||||
{{- range $key, $value := .Values.admin.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -130,7 +130,7 @@ spec:
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -203,9 +203,6 @@ spec:
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
- name: admin-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/admin/
|
||||
{{- end }}
|
||||
{{ tpl .Values.admin.extraVolumeMounts . | nindent 12 | trim }}
|
||||
ports:
|
||||
@@ -213,6 +210,10 @@ spec:
|
||||
name: http
|
||||
- containerPort: {{ .Values.admin.grpcPort }}
|
||||
name: grpc
|
||||
{{- if .Values.admin.metricsPort }}
|
||||
- containerPort: {{ .Values.admin.metricsPort }}
|
||||
name: metrics
|
||||
{{- end }}
|
||||
{{- if .Values.admin.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
@@ -297,9 +298,6 @@ spec:
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-client-cert
|
||||
- name: admin-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-admin-cert
|
||||
{{- end }}
|
||||
{{ tpl .Values.admin.extraVolumes . | indent 8 | trim }}
|
||||
{{- if .Values.admin.nodeSelector }}
|
||||
|
||||
@@ -104,7 +104,7 @@ spec:
|
||||
{{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 }}
|
||||
@@ -117,7 +117,7 @@ spec:
|
||||
{{- if and (ne $key $clusterMasterKey) (ne $key $clusterFilerKey) }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 }}
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-admin-cert
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: admin
|
||||
{{- if .Values.admin.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.admin.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
secretName: {{ template "seaweedfs.name" . }}-admin-cert
|
||||
issuerRef:
|
||||
name: {{ template "seaweedfs.name" . }}-ca-issuer
|
||||
kind: Issuer
|
||||
commonName: {{ .Values.certificates.commonName }}
|
||||
subject:
|
||||
organizations:
|
||||
- "SeaweedFS CA"
|
||||
dnsNames:
|
||||
- '*.{{ template "seaweedfs.name" . }}-admin'
|
||||
- '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}'
|
||||
- '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}.svc'
|
||||
- '*.{{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}.svc.cluster.local'
|
||||
{{- if .Values.certificates.ipAddresses }}
|
||||
ipAddresses:
|
||||
{{- range .Values.certificates.ipAddresses }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
privateKey:
|
||||
algorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
size: {{ .Values.certificates.keySize }}
|
||||
duration: {{ .Values.certificates.duration }}
|
||||
renewBefore: {{ .Values.certificates.renewBefore }}
|
||||
{{- end }}
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-worker-cert
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: worker
|
||||
{{- if .Values.worker.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.worker.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
secretName: {{ template "seaweedfs.name" . }}-worker-cert
|
||||
issuerRef:
|
||||
name: {{ template "seaweedfs.name" . }}-ca-issuer
|
||||
kind: Issuer
|
||||
commonName: {{ .Values.certificates.commonName }}
|
||||
subject:
|
||||
organizations:
|
||||
- "SeaweedFS CA"
|
||||
dnsNames:
|
||||
- '*.{{ template "seaweedfs.name" . }}-worker'
|
||||
- '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}'
|
||||
- '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}.svc'
|
||||
- '*.{{ template "seaweedfs.name" . }}-worker.{{ .Release.Namespace }}.svc.cluster.local'
|
||||
{{- if .Values.certificates.ipAddresses }}
|
||||
ipAddresses:
|
||||
{{- range .Values.certificates.ipAddresses }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
privateKey:
|
||||
algorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
size: {{ .Values.certificates.keySize }}
|
||||
duration: {{ .Values.certificates.duration }}
|
||||
renewBefore: {{ .Values.certificates.renewBefore }}
|
||||
{{- end }}
|
||||
|
||||
@@ -100,7 +100,7 @@ spec:
|
||||
{{- range $key, $value := .Values.cosi.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -111,7 +111,7 @@ spec:
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
|
||||
@@ -118,7 +118,7 @@ spec:
|
||||
{{- range $key, $value := .Values.filer.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -129,7 +129,7 @@ spec:
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -229,7 +229,7 @@ spec:
|
||||
- name: seaweedfs-filer-log-volume
|
||||
mountPath: "/logs/"
|
||||
{{- end }}
|
||||
{{- if and .Values.filer.s3.enabled .Values.filer.s3.enableAuth }}
|
||||
{{- if .Values.filer.s3.enableAuth }}
|
||||
- name: config-users
|
||||
mountPath: /etc/sw
|
||||
readOnly: true
|
||||
@@ -263,7 +263,7 @@ spec:
|
||||
mountPath: /usr/local/share/ca-certificates/filer/
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
{{- end }}
|
||||
{{ tpl .Values.filer.extraVolumeMounts . | nindent 12 | trim }}
|
||||
ports:
|
||||
@@ -281,18 +281,12 @@ spec:
|
||||
name: swfs-s3-tls
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $isJwtEnabled := or .Values.global.securityConfig.jwtSigning.filerWrite .Values.global.securityConfig.jwtSigning.filerRead }}
|
||||
{{- if .Values.filer.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
{{- if or $isJwtEnabled .Values.filer.readinessProbe.tcpSocket }}
|
||||
tcpSocket:
|
||||
port: {{ if $isJwtEnabled }}{{ .Values.filer.port }}{{ else }}{{ .Values.filer.readinessProbe.tcpSocket.port }}{{ end }}
|
||||
{{- else }}
|
||||
httpGet:
|
||||
path: {{ .Values.filer.readinessProbe.httpGet.path }}
|
||||
port: {{ .Values.filer.port }}
|
||||
scheme: {{ .Values.filer.readinessProbe.httpGet.scheme }}
|
||||
{{- end }}
|
||||
initialDelaySeconds: {{ .Values.filer.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.filer.readinessProbe.periodSeconds }}
|
||||
successThreshold: {{ .Values.filer.readinessProbe.successThreshold }}
|
||||
@@ -301,15 +295,10 @@ spec:
|
||||
{{- end }}
|
||||
{{- if .Values.filer.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
{{- if or $isJwtEnabled .Values.filer.livenessProbe.tcpSocket }}
|
||||
tcpSocket:
|
||||
port: {{ if $isJwtEnabled }}{{ .Values.filer.port }}{{ else }}{{ .Values.filer.livenessProbe.tcpSocket.port }}{{ end }}
|
||||
{{- else }}
|
||||
httpGet:
|
||||
path: {{ .Values.filer.livenessProbe.httpGet.path }}
|
||||
port: {{ .Values.filer.port }}
|
||||
scheme: {{ .Values.filer.livenessProbe.httpGet.scheme }}
|
||||
{{- end }}
|
||||
initialDelaySeconds: {{ .Values.filer.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.filer.livenessProbe.periodSeconds }}
|
||||
successThreshold: {{ .Values.filer.livenessProbe.successThreshold }}
|
||||
|
||||
@@ -102,7 +102,7 @@ spec:
|
||||
{{- range $key, $value := .Values.master.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -113,7 +113,7 @@ spec:
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
|
||||
@@ -94,7 +94,7 @@ spec:
|
||||
{{- range $key, $value := .Values.s3.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -105,7 +105,7 @@ spec:
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
|
||||
@@ -4,13 +4,6 @@
|
||||
{{- /* Determine service name based on deployment mode */}}
|
||||
{{- $serviceName := ternary (printf "%s-all-in-one" (include "seaweedfs.name" .)) (printf "%s-s3" (include "seaweedfs.name" .)) .Values.allInOne.enabled }}
|
||||
{{- $s3Port := .Values.allInOne.s3.port | default .Values.s3.port }}
|
||||
{{- /* Build hosts list - support both legacy .host (string) and new .hosts (array) for backwards compatibility */}}
|
||||
{{- $hosts := list }}
|
||||
{{- if kindIs "slice" .Values.s3.ingress.host }}
|
||||
{{- $hosts = .Values.s3.ingress.host }}
|
||||
{{- else if .Values.s3.ingress.host }}
|
||||
{{- $hosts = list .Values.s3.ingress.host }}
|
||||
{{- end }}
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
@@ -37,25 +30,6 @@ spec:
|
||||
tls:
|
||||
{{ .Values.s3.ingress.tls | default list | toYaml | nindent 6}}
|
||||
rules:
|
||||
{{- if $hosts }}
|
||||
{{- range $hosts }}
|
||||
- host: {{ . | quote }}
|
||||
http:
|
||||
paths:
|
||||
- path: {{ $.Values.s3.ingress.path | quote }}
|
||||
pathType: {{ $.Values.s3.ingress.pathType | quote }}
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ $serviceName }}
|
||||
port:
|
||||
number: {{ $s3Port }}
|
||||
{{- else }}
|
||||
serviceName: {{ $serviceName }}
|
||||
servicePort: {{ $s3Port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- else }}
|
||||
- http:
|
||||
paths:
|
||||
- path: {{ .Values.s3.ingress.path | quote }}
|
||||
@@ -70,5 +44,7 @@ spec:
|
||||
serviceName: {{ $serviceName }}
|
||||
servicePort: {{ $s3Port }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.ingress.host }}
|
||||
host: {{ .Values.s3.ingress.host | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
@@ -94,7 +94,7 @@ spec:
|
||||
{{- range $key, $value := .Values.sftp.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -105,7 +105,7 @@ spec:
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
|
||||
@@ -65,14 +65,6 @@ data:
|
||||
cert = "/usr/local/share/ca-certificates/filer/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/filer/tls.key"
|
||||
|
||||
[grpc.admin]
|
||||
cert = "/usr/local/share/ca-certificates/admin/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/admin/tls.key"
|
||||
|
||||
[grpc.worker]
|
||||
cert = "/usr/local/share/ca-certificates/worker/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/worker/tls.key"
|
||||
|
||||
# use this for any place needs a grpc client
|
||||
# i.e., "weed backup|benchmark|filer.copy|filer.replicate|mount|s3|upload"
|
||||
[grpc.client]
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
{{- if and .Values.volume.enabled .Values.volume.ingress.enabled }}
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: networking.k8s.io/v1beta1
|
||||
{{- else }}
|
||||
apiVersion: extensions/v1beta1
|
||||
{{- end }}
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-{{ template "seaweedfs.name" . }}-volume
|
||||
namespace: {{ .Release.Namespace }}
|
||||
annotations:
|
||||
{{- if and (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) .Values.volume.ingress.className }}
|
||||
kubernetes.io/ingress.class: {{ .Values.volume.ingress.className }}
|
||||
{{- end }}
|
||||
{{- with .Values.volume.ingress.annotations }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: volume
|
||||
spec:
|
||||
{{- if and (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) .Values.volume.ingress.className }}
|
||||
ingressClassName: {{ .Values.volume.ingress.className | quote }}
|
||||
{{- end }}
|
||||
tls:
|
||||
{{ .Values.volume.ingress.tls | default list | toYaml | nindent 6}}
|
||||
rules:
|
||||
- {{- if .Values.volume.ingress.host }}
|
||||
host: {{ .Values.volume.ingress.host | quote }}
|
||||
{{- end }}
|
||||
http:
|
||||
paths:
|
||||
- path: {{ .Values.volume.ingress.path | quote }}
|
||||
{{- if semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
pathType: {{ .Values.volume.ingress.pathType | quote }}
|
||||
{{- end }}
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ template "seaweedfs.name" . }}-volume
|
||||
port:
|
||||
number: {{ .Values.volume.port }}
|
||||
{{- else }}
|
||||
serviceName: {{ template "seaweedfs.name" . }}-volume
|
||||
servicePort: {{ .Values.volume.port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -58,11 +58,11 @@ metadata:
|
||||
helm.sh/hook-weight: "0"
|
||||
helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: {{ $seaweedfsName }}-volume-resize-hook
|
||||
restartPolicy: Never
|
||||
backoffLimit: 1
|
||||
containers:
|
||||
- name: resize
|
||||
image: {{ .Values.volume.resizeHook.image }}
|
||||
|
||||
@@ -122,7 +122,7 @@ spec:
|
||||
{{- range $key, $value := $volume.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -133,7 +133,7 @@ spec:
|
||||
{{- range $key, $value := $.Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -176,9 +176,6 @@ spec:
|
||||
{{- if $volume.dataCenter }}
|
||||
-dataCenter={{ $volume.dataCenter }} \
|
||||
{{- end }}
|
||||
{{- if $volume.id }}
|
||||
-id={{ $volume.id }} \
|
||||
{{- end }}
|
||||
-ip.bind={{ $volume.ipBind }} \
|
||||
-readMode={{ $volume.readMode }} \
|
||||
{{- if $volume.whiteList }}
|
||||
|
||||
@@ -97,7 +97,7 @@ spec:
|
||||
{{- range $key, $value := .Values.worker.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -108,7 +108,7 @@ spec:
|
||||
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ tpl $value $ | quote }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
@@ -134,14 +134,11 @@ spec:
|
||||
{{- if .Values.worker.adminServer }}
|
||||
-admin={{ .Values.worker.adminServer }} \
|
||||
{{- else }}
|
||||
-admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }}{{ if .Values.admin.grpcPort }}.{{ .Values.admin.grpcPort }}{{ end }} \
|
||||
-admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.grpcPort }} \
|
||||
{{- end }}
|
||||
-capabilities={{ .Values.worker.capabilities }} \
|
||||
-maxConcurrent={{ .Values.worker.maxConcurrent }} \
|
||||
-workingDir={{ .Values.worker.workingDir }}{{- if or .Values.worker.metricsPort .Values.worker.extraArgs }} \{{ end }}
|
||||
{{- if .Values.worker.metricsPort }}
|
||||
-metricsPort={{ .Values.worker.metricsPort }}{{- if .Values.worker.extraArgs }} \{{ end }}
|
||||
{{- end }}
|
||||
-workingDir={{ .Values.worker.workingDir }}{{- if .Values.worker.extraArgs }} \{{ end }}
|
||||
{{- range $index, $arg := .Values.worker.extraArgs }}
|
||||
{{ $arg }}{{- if lt $index (sub (len $.Values.worker.extraArgs) 1) }} \{{ end }}
|
||||
{{- end }}
|
||||
@@ -174,9 +171,6 @@ spec:
|
||||
- name: client-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/client/
|
||||
- name: worker-cert
|
||||
readOnly: true
|
||||
mountPath: /usr/local/share/ca-certificates/worker/
|
||||
{{- end }}
|
||||
{{ tpl .Values.worker.extraVolumeMounts . | nindent 12 | trim }}
|
||||
ports:
|
||||
@@ -190,13 +184,9 @@ spec:
|
||||
{{- end }}
|
||||
{{- if .Values.worker.livenessProbe.enabled }}
|
||||
livenessProbe:
|
||||
{{- if .Values.worker.livenessProbe.httpGet }}
|
||||
httpGet:
|
||||
path: {{ .Values.worker.livenessProbe.httpGet.path }}
|
||||
port: {{ .Values.worker.livenessProbe.httpGet.port }}
|
||||
{{- else if .Values.worker.livenessProbe.tcpSocket }}
|
||||
{{- with .Values.worker.livenessProbe.tcpSocket }}
|
||||
tcpSocket:
|
||||
port: {{ .Values.worker.livenessProbe.tcpSocket.port }}
|
||||
port: {{ .port }}
|
||||
{{- end }}
|
||||
initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }}
|
||||
@@ -206,13 +196,9 @@ spec:
|
||||
{{- end }}
|
||||
{{- if .Values.worker.readinessProbe.enabled }}
|
||||
readinessProbe:
|
||||
{{- if .Values.worker.readinessProbe.httpGet }}
|
||||
httpGet:
|
||||
path: {{ .Values.worker.readinessProbe.httpGet.path }}
|
||||
port: {{ .Values.worker.readinessProbe.httpGet.port }}
|
||||
{{- else if .Values.worker.readinessProbe.tcpSocket }}
|
||||
{{- with .Values.worker.readinessProbe.tcpSocket }}
|
||||
tcpSocket:
|
||||
port: {{ .Values.worker.readinessProbe.tcpSocket.port }}
|
||||
port: {{ .port }}
|
||||
{{- end }}
|
||||
initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
|
||||
@@ -276,9 +262,6 @@ spec:
|
||||
- name: client-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-client-cert
|
||||
- name: worker-cert
|
||||
secret:
|
||||
secretName: {{ template "seaweedfs.name" . }}-worker-cert
|
||||
{{- end }}
|
||||
{{ tpl .Values.worker.extraVolumes . | indent 8 | trim }}
|
||||
{{- if .Values.worker.nodeSelector }}
|
||||
|
||||
@@ -39,8 +39,8 @@ global:
|
||||
replicationPlacement: "001"
|
||||
extraEnvironmentVars:
|
||||
WEED_CLUSTER_DEFAULT: "sw"
|
||||
WEED_CLUSTER_SW_MASTER: "{{ include \"seaweedfs.cluster.masterAddress\" . }}"
|
||||
WEED_CLUSTER_SW_FILER: "{{ include \"seaweedfs.cluster.filerAddress\" . }}"
|
||||
WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
|
||||
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client.seaweedfs:8888"
|
||||
# WEED_JWT_SIGNING_KEY:
|
||||
# secretKeyRef:
|
||||
# name: seaweedfs-signing-key
|
||||
@@ -261,10 +261,10 @@ master:
|
||||
tls: []
|
||||
|
||||
extraEnvironmentVars:
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_1: "7"
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_2: "6"
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_3: "3"
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: "1"
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_1: '7'
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_2: '6'
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_3: '3'
|
||||
WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: '1'
|
||||
|
||||
# used to configure livenessProbe on master-server containers
|
||||
#
|
||||
@@ -348,16 +348,16 @@ volume:
|
||||
# maxVolumes: 0 # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
|
||||
|
||||
dataDirs:
|
||||
- name: data1
|
||||
type: "hostPath"
|
||||
hostPathPrefix: /ssd
|
||||
maxVolumes: 0
|
||||
- name: data1
|
||||
type: "hostPath"
|
||||
hostPathPrefix: /ssd
|
||||
maxVolumes: 0
|
||||
|
||||
# - name: data2
|
||||
# type: "persistentVolumeClaim"
|
||||
# storageClass: "yourClassNameOfChoice"
|
||||
# size: "800Gi"
|
||||
# maxVolumes: 0
|
||||
# - name: data2
|
||||
# type: "persistentVolumeClaim"
|
||||
# storageClass: "yourClassNameOfChoice"
|
||||
# size: "800Gi"
|
||||
# maxVolumes: 0
|
||||
|
||||
# This will automatically create a job for patching Kubernetes resources if the dataDirs type is 'persistentVolumeClaim' and the size has changed.
|
||||
resizeHook:
|
||||
@@ -397,13 +397,10 @@ volume:
|
||||
# limit background compaction or copying speed in mega bytes per second
|
||||
compactionMBps: "50"
|
||||
|
||||
|
||||
# Volume server's rack name
|
||||
rack: null
|
||||
|
||||
# Stable identifier for the volume server, independent of IP address
|
||||
# Useful for Kubernetes environments with hostPath volumes to maintain stable identity
|
||||
id: null
|
||||
|
||||
# Volume server's data center name
|
||||
dataCenter: null
|
||||
|
||||
@@ -544,30 +541,6 @@ volume:
|
||||
failureThreshold: 100
|
||||
timeoutSeconds: 30
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
className: ""
|
||||
host: "volume.seaweedfs.local"
|
||||
path: "/"
|
||||
pathType: Prefix
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/app-root: /ui/index.html
|
||||
# nginx.ingress.kubernetes.io/use-regex: "true"
|
||||
# nginx.ingress.kubernetes.io/rewrite-target: /$1
|
||||
# nginx.ingress.kubernetes.io/auth-type: "basic"
|
||||
# nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
|
||||
# nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Volume'
|
||||
# nginx.ingress.kubernetes.io/service-upstream: "true"
|
||||
# nginx.ingress.kubernetes.io/enable-rewrite-log: "true"
|
||||
# nginx.ingress.kubernetes.io/ssl-redirect: "false"
|
||||
# nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
|
||||
# nginx.ingress.kubernetes.io/configuration-snippet: |
|
||||
# sub_filter '<head>' '<head> <base href="/sw-volume/">'; #add base url
|
||||
# sub_filter '="/' '="./'; #make absolute paths to relative
|
||||
# sub_filter '=/' '=./';
|
||||
# sub_filter '/seaweedfsstatic' './seaweedfsstatic';
|
||||
# sub_filter_once off;
|
||||
|
||||
# Map of named volume groups for topology-aware deployments.
|
||||
# Each key inherits all fields from the `volume` section but can override
|
||||
# them locally—for example, replicas, nodeSelector, dataCenter, etc.
|
||||
@@ -869,14 +842,14 @@ filer:
|
||||
|
||||
# secret env variables
|
||||
secretExtraEnvironmentVars: {}
|
||||
# WEED_POSTGRES_USERNAME:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
# key: username
|
||||
# WEED_POSTGRES_PASSWORD:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
# key: password
|
||||
# WEED_POSTGRES_USERNAME:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
# key: username
|
||||
# WEED_POSTGRES_PASSWORD:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
# key: password
|
||||
|
||||
s3:
|
||||
enabled: false
|
||||
@@ -1035,7 +1008,7 @@ s3:
|
||||
ingress:
|
||||
enabled: false
|
||||
className: ""
|
||||
# host: false for "*" hostname, or an array for multiple hostnames
|
||||
# host: false for "*" hostname
|
||||
host: "seaweedfs.cluster.local"
|
||||
path: "/"
|
||||
pathType: Prefix
|
||||
@@ -1122,6 +1095,7 @@ admin:
|
||||
replicas: 1
|
||||
port: 23646 # Default admin port
|
||||
grpcPort: 33646 # Default gRPC port for worker connections
|
||||
metricsPort: 9327
|
||||
loggingOverrideLevel: null
|
||||
|
||||
# Admin authentication
|
||||
@@ -1253,6 +1227,7 @@ worker:
|
||||
metricsPort: 9327
|
||||
|
||||
# Admin server to connect to
|
||||
# Format: "host:port" or auto-discover from admin service
|
||||
adminServer: ""
|
||||
|
||||
# Worker capabilities - comma-separated list
|
||||
@@ -1327,11 +1302,10 @@ worker:
|
||||
extraEnvironmentVars: {}
|
||||
|
||||
# Health checks for worker pods
|
||||
# Workers expose /health (liveness) and /ready (readiness) endpoints on the metricsPort
|
||||
# Since workers do not have an HTTP endpoint, a tcpSocket probe on the metrics port is recommended.
|
||||
livenessProbe:
|
||||
enabled: true
|
||||
httpGet:
|
||||
path: /health
|
||||
tcpSocket:
|
||||
port: metrics
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 60
|
||||
@@ -1341,8 +1315,7 @@ worker:
|
||||
|
||||
readinessProbe:
|
||||
enabled: true
|
||||
httpGet:
|
||||
path: /ready
|
||||
tcpSocket:
|
||||
port: metrics
|
||||
initialDelaySeconds: 20
|
||||
periodSeconds: 15
|
||||
|
||||
|
Before Width: | Height: | Size: 71 KiB After Width: | Height: | Size: 90 KiB |
|
Before Width: | Height: | Size: 78 KiB After Width: | Height: | Size: 95 KiB |
|
Before Width: | Height: | Size: 110 KiB After Width: | Height: | Size: 127 KiB |
|
Before Width: | Height: | Size: 67 KiB After Width: | Height: | Size: 81 KiB |
@@ -87,7 +87,7 @@ public class FilerGrpcClient {
|
||||
.maxHeaderListSize(16 * 1024 * 1024)
|
||||
.keepAliveTime(KEEP_ALIVE_TIME_SECONDS, TimeUnit.SECONDS)
|
||||
.keepAliveTimeout(KEEP_ALIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS)
|
||||
.keepAliveWithoutCalls(false)
|
||||
.keepAliveWithoutCalls(true)
|
||||
.withOption(io.grpc.netty.shaded.io.netty.channel.ChannelOption.SO_RCVBUF, 16 * 1024 * 1024)
|
||||
.withOption(io.grpc.netty.shaded.io.netty.channel.ChannelOption.SO_SNDBUF, 16 * 1024 * 1024);
|
||||
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
# SeaweedFS S3 Java SDK Compatibility Tests
|
||||
|
||||
This project contains Java-based integration tests for SeaweedFS S3 API compatibility.
|
||||
|
||||
## Overview
|
||||
|
||||
Tests are provided for both AWS SDK v1 and v2 to ensure compatibility with the various SDK versions commonly used in production.
|
||||
|
||||
## SDK Versions
|
||||
|
||||
| SDK | Version | Notes |
|
||||
|-----|---------|-------|
|
||||
| AWS SDK v1 for Java | 1.12.600 | Legacy SDK, less strict ETag validation |
|
||||
| AWS SDK v2 for Java | 2.20.127 | Modern SDK with strict checksum validation |
|
||||
|
||||
## Running Tests
|
||||
|
||||
### Prerequisites
|
||||
|
||||
1. SeaweedFS running with S3 API enabled:
|
||||
```bash
|
||||
weed server -s3
|
||||
```
|
||||
|
||||
2. Java 18+ and Maven
|
||||
|
||||
### Run All Tests
|
||||
|
||||
```bash
|
||||
mvn test
|
||||
```
|
||||
|
||||
### Run Specific Tests
|
||||
|
||||
```bash
|
||||
# Run only ETag validation tests (AWS SDK v2)
|
||||
mvn test -Dtest=ETagValidationTest
|
||||
|
||||
# Run with custom endpoint
|
||||
mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `S3_ENDPOINT` | `http://127.0.0.1:8333` | S3 API endpoint URL |
|
||||
| `S3_ACCESS_KEY` | `some_access_key1` | Access key ID |
|
||||
| `S3_SECRET_KEY` | `some_secret_key1` | Secret access key |
|
||||
| `S3_REGION` | `us-east-1` | AWS region |
|
||||
|
||||
## Test Coverage
|
||||
|
||||
### ETagValidationTest (AWS SDK v2)
|
||||
|
||||
Tests for [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768) - ETag format validation.
|
||||
|
||||
| Test | Description |
|
||||
|------|-------------|
|
||||
| `testSmallFilePutObject` | Verify small files return pure MD5 ETag |
|
||||
| `testLargeFilePutObject_Issue7768` | **Critical**: Verify large files (>8MB) return pure MD5 ETag |
|
||||
| `testExtraLargeFilePutObject` | Verify very large files (>24MB) return pure MD5 ETag |
|
||||
| `testMultipartUploadETag` | Verify multipart uploads return composite ETag |
|
||||
| `testETagConsistency` | Verify ETag consistency across PUT/HEAD/GET |
|
||||
| `testMultipleLargeFileUploads` | Stress test multiple large uploads |
|
||||
|
||||
### Background: Issue #7768
|
||||
|
||||
AWS SDK v2 for Java includes checksum validation that decodes the ETag as hexadecimal. When SeaweedFS returned composite ETags (`<md5>-<count>`) for regular `PutObject` with internally auto-chunked files, the SDK failed with:
|
||||
|
||||
```
|
||||
java.lang.IllegalArgumentException: Invalid base 16 character: '-'
|
||||
```
|
||||
|
||||
**Per AWS S3 specification:**
|
||||
- `PutObject`: ETag is always a pure MD5 hex string (32 chars)
|
||||
- `CompleteMultipartUpload`: ETag is composite format (`<md5>-<partcount>`)
|
||||
|
||||
The fix ensures SeaweedFS follows this specification.
|
||||
|
||||
## Project Structure
|
||||
|
||||
```
|
||||
src/
|
||||
├── main/java/com/seaweedfs/s3/
|
||||
│ ├── PutObject.java # Example PutObject with SDK v1
|
||||
│ └── HighLevelMultipartUpload.java
|
||||
└── test/java/com/seaweedfs/s3/
|
||||
├── PutObjectTest.java # Basic SDK v1 test
|
||||
└── ETagValidationTest.java # Comprehensive SDK v2 ETag tests
|
||||
```
|
||||
|
||||
## Validated SDK Versions
|
||||
|
||||
This Java test project validates:
|
||||
|
||||
- ✅ AWS SDK v2 for Java 2.20.127+
|
||||
- ✅ AWS SDK v1 for Java 1.12.600+
|
||||
|
||||
Go SDK validation is performed by separate test suites:
|
||||
- See [Go ETag Tests](/test/s3/etag/) for AWS SDK v2 for Go tests
|
||||
- See [test/s3/SDK_COMPATIBILITY.md](/test/s3/SDK_COMPATIBILITY.md) for full SDK compatibility matrix
|
||||
|
||||
## Related
|
||||
|
||||
- [GitHub Issue #7768](https://github.com/seaweedfs/seaweedfs/issues/7768)
|
||||
- [AWS S3 ETag Documentation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html)
|
||||
- [Go ETag Tests](/test/s3/etag/)
|
||||
- [SDK Compatibility Matrix](/test/s3/SDK_COMPATIBILITY.md)
|
||||
|
||||
@@ -6,29 +6,18 @@
|
||||
<packaging>jar</packaging>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<aws.sdk.v1.version>1.12.600</aws.sdk.v1.version>
|
||||
<aws.sdk.v2.version>2.20.127</aws.sdk.v2.version>
|
||||
<maven.compiler.source>18</maven.compiler.source>
|
||||
<maven.compiler.target>18</maven.compiler.target>
|
||||
</properties>
|
||||
<name>copier</name>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<!-- AWS SDK v1 BOM -->
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-bom</artifactId>
|
||||
<version>${aws.sdk.v1.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
<!-- AWS SDK v2 BOM -->
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>bom</artifactId>
|
||||
<version>${aws.sdk.v2.version}</version>
|
||||
<version>1.11.327</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
@@ -36,50 +25,15 @@
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<!-- AWS SDK v1 (for backward compatibility with existing code) -->
|
||||
<dependency>
|
||||
<groupId>com.amazonaws</groupId>
|
||||
<artifactId>aws-java-sdk-s3</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- AWS SDK v2 (for modern tests with checksum validation) -->
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>s3</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>s3-transfer-manager</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- Test dependencies -->
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.13.2</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<version>5.10.0</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<version>3.24.2</version>
|
||||
<version>4.13.1</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>3.2.2</version>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
|
||||
@@ -1,439 +0,0 @@
|
||||
package com.seaweedfs.s3;
|
||||
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.regions.Region;
|
||||
import software.amazon.awssdk.services.s3.S3Client;
|
||||
import software.amazon.awssdk.services.s3.model.*;
|
||||
|
||||
import java.net.URI;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.assertj.core.api.Assertions.*;
|
||||
|
||||
/**
|
||||
* AWS SDK v2 Integration Tests for S3 ETag Format Validation.
|
||||
*
|
||||
* These tests verify that SeaweedFS returns correct ETag formats that are
|
||||
* compatible with AWS SDK v2's checksum validation.
|
||||
*
|
||||
* Background (GitHub Issue #7768):
|
||||
* AWS SDK v2 for Java validates ETags as hexadecimal MD5 hashes for PutObject
|
||||
* responses. If the ETag contains non-hex characters (like '-' in composite
|
||||
* format), the SDK fails with "Invalid base 16 character: '-'".
|
||||
*
|
||||
* Per AWS S3 specification:
|
||||
* - Regular PutObject: ETag is always a pure MD5 hex string (32 chars)
|
||||
* - CompleteMultipartUpload: ETag is composite format "<md5>-<partcount>"
|
||||
*
|
||||
* To run these tests:
|
||||
* mvn test -Dtest=ETagValidationTest -DS3_ENDPOINT=http://localhost:8333
|
||||
*
|
||||
* Or set environment variable:
|
||||
* export S3_ENDPOINT=http://localhost:8333
|
||||
* mvn test -Dtest=ETagValidationTest
|
||||
*/
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
@DisplayName("S3 ETag Format Validation Tests (AWS SDK v2)")
|
||||
class ETagValidationTest {
|
||||
|
||||
// Configuration - can be overridden via system properties or environment variables
|
||||
private static final String DEFAULT_ENDPOINT = "http://127.0.0.1:8333";
|
||||
private static final String DEFAULT_ACCESS_KEY = "some_access_key1";
|
||||
private static final String DEFAULT_SECRET_KEY = "some_secret_key1";
|
||||
private static final String DEFAULT_REGION = "us-east-1";
|
||||
|
||||
// Auto-chunking threshold in SeaweedFS (must match s3api_object_handlers_put.go)
|
||||
private static final int AUTO_CHUNK_SIZE = 8 * 1024 * 1024; // 8MB
|
||||
|
||||
// Test sizes
|
||||
private static final int SMALL_FILE_SIZE = 1024; // 1KB
|
||||
private static final int LARGE_FILE_SIZE = 10 * 1024 * 1024; // 10MB (triggers auto-chunking)
|
||||
private static final int XL_FILE_SIZE = 25 * 1024 * 1024; // 25MB (multiple chunks)
|
||||
private static final int MULTIPART_PART_SIZE = 5 * 1024 * 1024; // 5MB per part
|
||||
|
||||
// ETag format patterns
|
||||
private static final Pattern PURE_MD5_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}\"?$");
|
||||
private static final Pattern COMPOSITE_PATTERN = Pattern.compile("^\"?[a-f0-9]{32}-\\d+\"?$");
|
||||
|
||||
private S3Client s3Client;
|
||||
private String testBucketName;
|
||||
private final SecureRandom random = new SecureRandom();
|
||||
|
||||
@BeforeAll
|
||||
void setUp() {
|
||||
String endpoint = getConfig("S3_ENDPOINT", DEFAULT_ENDPOINT);
|
||||
String accessKey = getConfig("S3_ACCESS_KEY", DEFAULT_ACCESS_KEY);
|
||||
String secretKey = getConfig("S3_SECRET_KEY", DEFAULT_SECRET_KEY);
|
||||
String region = getConfig("S3_REGION", DEFAULT_REGION);
|
||||
|
||||
System.out.println("Connecting to S3 endpoint: " + endpoint);
|
||||
|
||||
s3Client = S3Client.builder()
|
||||
.endpointOverride(URI.create(endpoint))
|
||||
.credentialsProvider(StaticCredentialsProvider.create(
|
||||
AwsBasicCredentials.create(accessKey, secretKey)))
|
||||
.region(Region.of(region))
|
||||
.forcePathStyle(true) // Required for SeaweedFS
|
||||
.build();
|
||||
|
||||
// Create test bucket
|
||||
testBucketName = "test-etag-" + UUID.randomUUID().toString().substring(0, 8);
|
||||
s3Client.createBucket(CreateBucketRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.build());
|
||||
|
||||
System.out.println("Created test bucket: " + testBucketName);
|
||||
}
|
||||
|
||||
@AfterAll
|
||||
void tearDown() {
|
||||
if (s3Client != null && testBucketName != null) {
|
||||
try {
|
||||
// Delete all objects with pagination
|
||||
String continuationToken = null;
|
||||
do {
|
||||
ListObjectsV2Response listResp = s3Client.listObjectsV2(
|
||||
ListObjectsV2Request.builder()
|
||||
.bucket(testBucketName)
|
||||
.continuationToken(continuationToken)
|
||||
.build());
|
||||
for (S3Object obj : listResp.contents()) {
|
||||
s3Client.deleteObject(DeleteObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(obj.key())
|
||||
.build());
|
||||
}
|
||||
continuationToken = listResp.nextContinuationToken();
|
||||
} while (continuationToken != null);
|
||||
|
||||
// Abort any multipart uploads
|
||||
ListMultipartUploadsResponse mpResp = s3Client.listMultipartUploads(
|
||||
ListMultipartUploadsRequest.builder().bucket(testBucketName).build());
|
||||
for (MultipartUpload upload : mpResp.uploads()) {
|
||||
s3Client.abortMultipartUpload(AbortMultipartUploadRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(upload.key())
|
||||
.uploadId(upload.uploadId())
|
||||
.build());
|
||||
}
|
||||
|
||||
// Delete bucket
|
||||
s3Client.deleteBucket(DeleteBucketRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.build());
|
||||
|
||||
System.out.println("Cleaned up test bucket: " + testBucketName);
|
||||
} catch (Exception e) {
|
||||
System.err.println("Warning: Failed to cleanup test bucket: " + e.getMessage());
|
||||
}
|
||||
s3Client.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Small file PutObject should return pure MD5 ETag")
|
||||
void testSmallFilePutObject() throws Exception {
|
||||
byte[] testData = generateRandomData(SMALL_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "small-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
System.out.println("Small file ETag: " + etag + " (expected MD5: " + expectedMD5 + ")");
|
||||
|
||||
assertThat(etag)
|
||||
.describedAs("Small file ETag should be pure MD5")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("ETag should match calculated MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
assertThat(etag)
|
||||
.describedAs("ETag should not contain hyphen")
|
||||
.doesNotContain("-");
|
||||
}
|
||||
|
||||
/**
|
||||
* Critical test for GitHub Issue #7768.
|
||||
*
|
||||
* This test uploads a file larger than the auto-chunking threshold (8MB),
|
||||
* which triggers SeaweedFS to split the file into multiple internal chunks.
|
||||
*
|
||||
* Previously, this caused SeaweedFS to return a composite ETag like
|
||||
* "d41d8cd98f00b204e9800998ecf8427e-2", which AWS SDK v2 rejected because
|
||||
* it validates the ETag as hexadecimal and '-' is not a valid hex character.
|
||||
*
|
||||
* The fix ensures that regular PutObject always returns a pure MD5 ETag,
|
||||
* regardless of internal chunking.
|
||||
*/
|
||||
@Test
|
||||
@DisplayName("Large file PutObject (>8MB) should return pure MD5 ETag - Issue #7768")
|
||||
void testLargeFilePutObject_Issue7768() throws Exception {
|
||||
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "large-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
System.out.println("Uploading large file (" + LARGE_FILE_SIZE + " bytes, " +
|
||||
"> " + AUTO_CHUNK_SIZE + " byte auto-chunk threshold)...");
|
||||
|
||||
// This is where Issue #7768 would manifest - SDK v2 validates ETag
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
int expectedChunks = (LARGE_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
|
||||
System.out.println("Large file ETag: " + etag +
|
||||
" (expected MD5: " + expectedMD5 + ", internal chunks: ~" + expectedChunks + ")");
|
||||
|
||||
// These assertions would fail before the fix
|
||||
assertThat(etag)
|
||||
.describedAs("Large file PutObject ETag MUST be pure MD5 (not composite)")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(etag)
|
||||
.describedAs("Large file ETag should NOT be composite format")
|
||||
.doesNotMatch(COMPOSITE_PATTERN);
|
||||
assertThat(etag)
|
||||
.describedAs("ETag should not contain hyphen for regular PutObject")
|
||||
.doesNotContain("-");
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("ETag should match calculated MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
|
||||
// Verify hex decoding works (this is what fails in Issue #7768)
|
||||
assertThatCode(() -> hexToBytes(cleanETag(etag)))
|
||||
.describedAs("ETag should be valid hexadecimal (AWS SDK v2 validation)")
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Extra large file PutObject (>24MB) should return pure MD5 ETag")
|
||||
void testExtraLargeFilePutObject() throws Exception {
|
||||
byte[] testData = generateRandomData(XL_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "xl-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
int expectedChunks = (XL_FILE_SIZE / AUTO_CHUNK_SIZE) + 1;
|
||||
System.out.println("Uploading XL file (" + XL_FILE_SIZE + " bytes, ~" +
|
||||
expectedChunks + " internal chunks)...");
|
||||
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
System.out.println("XL file ETag: " + etag);
|
||||
|
||||
assertThat(etag)
|
||||
.describedAs("XL file PutObject ETag MUST be pure MD5")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("ETag should match calculated MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Multipart upload should return composite ETag")
|
||||
void testMultipartUploadETag() throws Exception {
|
||||
int totalSize = 15 * 1024 * 1024; // 15MB = 3 parts
|
||||
byte[] testData = generateRandomData(totalSize);
|
||||
String objectKey = "multipart-file-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
System.out.println("Performing multipart upload (" + totalSize + " bytes)...");
|
||||
|
||||
// Initiate multipart upload
|
||||
CreateMultipartUploadResponse createResp = s3Client.createMultipartUpload(
|
||||
CreateMultipartUploadRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build());
|
||||
String uploadId = createResp.uploadId();
|
||||
|
||||
List<CompletedPart> completedParts = new ArrayList<>();
|
||||
int partNumber = 1;
|
||||
|
||||
// Upload parts
|
||||
for (int offset = 0; offset < totalSize; offset += MULTIPART_PART_SIZE) {
|
||||
int end = Math.min(offset + MULTIPART_PART_SIZE, totalSize);
|
||||
byte[] partData = new byte[end - offset];
|
||||
System.arraycopy(testData, offset, partData, 0, partData.length);
|
||||
|
||||
UploadPartResponse uploadResp = s3Client.uploadPart(
|
||||
UploadPartRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.uploadId(uploadId)
|
||||
.partNumber(partNumber)
|
||||
.build(),
|
||||
RequestBody.fromBytes(partData));
|
||||
|
||||
completedParts.add(CompletedPart.builder()
|
||||
.partNumber(partNumber)
|
||||
.eTag(uploadResp.eTag())
|
||||
.build());
|
||||
partNumber++;
|
||||
}
|
||||
|
||||
// Complete multipart upload
|
||||
CompleteMultipartUploadResponse completeResp = s3Client.completeMultipartUpload(
|
||||
CompleteMultipartUploadRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.uploadId(uploadId)
|
||||
.multipartUpload(CompletedMultipartUpload.builder()
|
||||
.parts(completedParts)
|
||||
.build())
|
||||
.build());
|
||||
|
||||
String etag = completeResp.eTag();
|
||||
System.out.println("Multipart upload ETag: " + etag + " (" + completedParts.size() + " parts)");
|
||||
|
||||
// Multipart uploads SHOULD have composite ETag
|
||||
assertThat(etag)
|
||||
.describedAs("Multipart upload ETag SHOULD be composite format")
|
||||
.matches(COMPOSITE_PATTERN);
|
||||
assertThat(etag)
|
||||
.describedAs("Multipart ETag should contain hyphen")
|
||||
.contains("-");
|
||||
|
||||
// Verify part count in ETag
|
||||
String[] parts = cleanETag(etag).split("-");
|
||||
assertThat(parts).hasSize(2);
|
||||
assertThat(parts[1])
|
||||
.describedAs("Part count in ETag should match uploaded parts")
|
||||
.isEqualTo(String.valueOf(completedParts.size()));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("ETag should be consistent across PUT, HEAD, and GET")
|
||||
void testETagConsistency() throws Exception {
|
||||
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
|
||||
String objectKey = "consistency-test-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
// PUT
|
||||
PutObjectResponse putResp = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
String putETag = putResp.eTag();
|
||||
|
||||
// HEAD
|
||||
HeadObjectResponse headResp = s3Client.headObject(
|
||||
HeadObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build());
|
||||
String headETag = headResp.eTag();
|
||||
|
||||
// GET
|
||||
GetObjectResponse getResp = s3Client.getObject(
|
||||
GetObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build())
|
||||
.response();
|
||||
String getETag = getResp.eTag();
|
||||
|
||||
System.out.println("PUT ETag: " + putETag + ", HEAD ETag: " + headETag + ", GET ETag: " + getETag);
|
||||
|
||||
assertThat(putETag).isEqualTo(headETag);
|
||||
assertThat(putETag).isEqualTo(getETag);
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("Multiple large file uploads should all return pure MD5 ETags")
|
||||
void testMultipleLargeFileUploads() throws Exception {
|
||||
int numFiles = 3;
|
||||
|
||||
for (int i = 0; i < numFiles; i++) {
|
||||
byte[] testData = generateRandomData(LARGE_FILE_SIZE);
|
||||
String expectedMD5 = calculateMD5Hex(testData);
|
||||
String objectKey = "multi-large-" + i + "-" + UUID.randomUUID() + ".bin";
|
||||
|
||||
PutObjectResponse response = s3Client.putObject(
|
||||
PutObjectRequest.builder()
|
||||
.bucket(testBucketName)
|
||||
.key(objectKey)
|
||||
.build(),
|
||||
RequestBody.fromBytes(testData));
|
||||
|
||||
String etag = response.eTag();
|
||||
System.out.println("File " + i + " ETag: " + etag);
|
||||
|
||||
assertThat(etag)
|
||||
.describedAs("File " + i + " ETag should be pure MD5")
|
||||
.matches(PURE_MD5_PATTERN);
|
||||
assertThat(cleanETag(etag))
|
||||
.describedAs("File " + i + " ETag should match MD5")
|
||||
.isEqualTo(expectedMD5);
|
||||
|
||||
// Validate hex decoding
|
||||
assertThatCode(() -> hexToBytes(cleanETag(etag)))
|
||||
.doesNotThrowAnyException();
|
||||
}
|
||||
}
|
||||
|
||||
// Helper methods
|
||||
|
||||
private String getConfig(String key, String defaultValue) {
|
||||
String value = System.getProperty(key);
|
||||
if (value == null) {
|
||||
value = System.getenv(key);
|
||||
}
|
||||
return value != null ? value : defaultValue;
|
||||
}
|
||||
|
||||
private byte[] generateRandomData(int size) {
|
||||
byte[] data = new byte[size];
|
||||
random.nextBytes(data);
|
||||
return data;
|
||||
}
|
||||
|
||||
private String calculateMD5Hex(byte[] data) throws Exception {
|
||||
MessageDigest md = MessageDigest.getInstance("MD5");
|
||||
byte[] digest = md.digest(data);
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (byte b : digest) {
|
||||
sb.append(String.format("%02x", b));
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private String cleanETag(String etag) {
|
||||
if (etag == null) return null;
|
||||
return etag.replace("\"", "");
|
||||
}
|
||||
|
||||
private byte[] hexToBytes(String hex) {
|
||||
int len = hex.length();
|
||||
byte[] data = new byte[len / 2];
|
||||
for (int i = 0; i < len; i += 2) {
|
||||
data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4)
|
||||
+ Character.digit(hex.charAt(i + 1), 16));
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1319,99 +1319,6 @@
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 90,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 API Calls per Bucket",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "reqps",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
@@ -1425,8 +1332,8 @@
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
@@ -3053,200 +2960,6 @@
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 37
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 87,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}} (logical)",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}} (physical)",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 Bucket Size",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "decbytes",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 37
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 88,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 Bucket Object Count",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
|
||||
@@ -875,85 +875,6 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS-DEV}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 86,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"percentage": false,
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeShift": null,
|
||||
"title": "S3 API Calls per Bucket",
|
||||
"tooltip": {
|
||||
"msResolution": true,
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "cumulative"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "reqps",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
@@ -1166,202 +1087,6 @@
|
||||
"show": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 37
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 87,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"span": 6,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}} (logical)",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}} (physical)",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 Bucket Size",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "decbytes",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "${DS_PROMETHEUS}",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 37
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 88,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null as zero",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"span": 6,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 Bucket Object Count",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
}
|
||||
],
|
||||
"repeat": null,
|
||||
@@ -2386,4 +2111,4 @@
|
||||
"timezone": "browser",
|
||||
"title": "SeaweedFS ks8 heartbeat",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
|
||||
@@ -447,99 +447,6 @@
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
"dashLength": 10,
|
||||
"dashes": false,
|
||||
"datasource": "$DS_PROMETHEUS",
|
||||
"fill": 1,
|
||||
"fillGradient": 0,
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
"id": 86,
|
||||
"legend": {
|
||||
"avg": false,
|
||||
"current": false,
|
||||
"max": false,
|
||||
"min": false,
|
||||
"show": true,
|
||||
"total": false,
|
||||
"values": false
|
||||
},
|
||||
"lines": true,
|
||||
"linewidth": 1,
|
||||
"links": [],
|
||||
"nullPointMode": "null",
|
||||
"options": {
|
||||
"alertThreshold": true
|
||||
},
|
||||
"percentage": false,
|
||||
"pluginVersion": "8.1.2",
|
||||
"pointradius": 5,
|
||||
"points": false,
|
||||
"renderer": "flot",
|
||||
"seriesOverrides": [],
|
||||
"spaceLength": 10,
|
||||
"stack": false,
|
||||
"steppedLine": false,
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(rate(SeaweedFS_s3_request_total{namespace=\"$namespace\",service=~\"$service-api\"}[$__interval])) by (bucket)",
|
||||
"format": "time_series",
|
||||
"hide": false,
|
||||
"intervalFactor": 2,
|
||||
"legendFormat": "{{bucket}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"thresholds": [],
|
||||
"timeFrom": null,
|
||||
"timeRegions": [],
|
||||
"timeShift": null,
|
||||
"title": "S3 API Calls per Bucket",
|
||||
"tooltip": {
|
||||
"shared": true,
|
||||
"sort": 0,
|
||||
"value_type": "individual"
|
||||
},
|
||||
"type": "graph",
|
||||
"xaxis": {
|
||||
"buckets": null,
|
||||
"mode": "time",
|
||||
"name": null,
|
||||
"show": true,
|
||||
"values": []
|
||||
},
|
||||
"yaxes": [
|
||||
{
|
||||
"format": "reqps",
|
||||
"logBase": 1,
|
||||
"min": 0,
|
||||
"show": true
|
||||
},
|
||||
{
|
||||
"format": "short",
|
||||
"logBase": 1,
|
||||
"show": false
|
||||
}
|
||||
],
|
||||
"yaxis": {
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"aliasColors": {},
|
||||
"bars": false,
|
||||
@@ -559,8 +466,8 @@
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
@@ -670,8 +577,8 @@
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
@@ -781,8 +688,8 @@
|
||||
"grid": {},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 15
|
||||
},
|
||||
"hiddenSeries": false,
|
||||
@@ -2447,209 +2354,6 @@
|
||||
"align": false,
|
||||
"alignLevel": null
|
||||
}
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "decbytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 41
|
||||
},
|
||||
"id": 89,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}} (logical)",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_physical_size_bytes{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}} (physical)",
|
||||
"range": true,
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "S3 Bucket Size",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 80
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 41
|
||||
},
|
||||
"id": 90,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "10.3.1",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max(SeaweedFS_s3_bucket_object_count{namespace=\"$NAMESPACE\"}) by (bucket)",
|
||||
"legendFormat": "{{bucket}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "S3 Bucket Object Count",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
@@ -2841,4 +2545,4 @@
|
||||
"timezone": "browser",
|
||||
"title": "SeaweedFS",
|
||||
"version": 2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,14 +3,13 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/other/mq_client_example/example"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/client/agent_client"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/schema"
|
||||
"log"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/other/mq_client_example/example"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/client/agent_client"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/schema"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -3,14 +3,13 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/other/mq_client_example/example"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/client/agent_client"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module seaweedfs-rdma-sidecar
|
||||
|
||||
go 1.24.9
|
||||
go 1.24
|
||||
|
||||
require (
|
||||
github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
|
||||
@@ -18,27 +18,27 @@ require (
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/prometheus/client_golang v1.23.0 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.19.2 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/prometheus/common v0.65.0 // indirect
|
||||
github.com/prometheus/procfs v0.17.0 // indirect
|
||||
github.com/sagikazarmark/locafero v0.7.0 // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spf13/viper v1.21.0 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spf13/afero v1.12.0 // indirect
|
||||
github.com/spf13/cast v1.7.1 // indirect
|
||||
github.com/spf13/pflag v1.0.6 // indirect
|
||||
github.com/spf13/viper v1.20.1 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/image v0.34.0 // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
|
||||
google.golang.org/grpc v1.77.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
golang.org/x/image v0.30.0 // indirect
|
||||
golang.org/x/net v0.43.0 // indirect
|
||||
golang.org/x/sys v0.35.0 // indirect
|
||||
golang.org/x/text v0.28.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 // indirect
|
||||
google.golang.org/grpc v1.74.2 // indirect
|
||||
google.golang.org/protobuf v1.36.7 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
|
||||
|
||||
@@ -44,43 +44,34 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc=
|
||||
github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
|
||||
github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
|
||||
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
|
||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
|
||||
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
|
||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
|
||||
github.com/seaweedfs/goexif v1.0.3/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk=
|
||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
|
||||
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
|
||||
github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
|
||||
github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
|
||||
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
|
||||
github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
|
||||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
|
||||
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4=
|
||||
github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
@@ -109,26 +100,19 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
golang.org/x/image v0.30.0 h1:jD5RhkmVAnjqaCUXfbGBrn3lpxbknfN9w2UhHHU+5B4=
|
||||
golang.org/x/image v0.30.0/go.mod h1:SAEUTxCCMWSrJcCy/4HwavEsfZZJlYxeHLc6tTiAe/c=
|
||||
golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
|
||||
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
|
||||
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
|
||||
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
|
||||
golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0 h1:MAKi5q709QWfnkkpNQ0M12hYJ1+e8qYVDyowc4U1XZM=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250728155136-f173205681a0/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4=
|
||||
google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM=
|
||||
google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
|
||||
google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
|
||||
google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
|
||||
@@ -419,7 +419,7 @@ func (c *Client) Read(ctx context.Context, req *ReadRequest) (*ReadResponse, err
|
||||
}).Info("✅ RDMA read completed successfully")
|
||||
|
||||
// MOCK DATA IMPLEMENTATION - FOR DEVELOPMENT/TESTING ONLY
|
||||
//
|
||||
//
|
||||
// This section generates placeholder data for the mock RDMA implementation.
|
||||
// In a production RDMA implementation, this should be replaced with:
|
||||
//
|
||||
@@ -472,7 +472,7 @@ func (c *Client) ReadFileRange(ctx context.Context, fileID string, offset, size
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid file ID %s: %w", fileID, err)
|
||||
}
|
||||
|
||||
|
||||
req := &ReadRequest{
|
||||
VolumeID: volumeID,
|
||||
NeedleID: needleID,
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"seaweedfs-rdma-sidecar/pkg/rdma"
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
// Code generated by protoc-gen-go. DO NOT EDIT.
|
||||
// versions:
|
||||
// protoc-gen-go v1.36.6
|
||||
// protoc v6.33.1
|
||||
// source: telemetry/proto/telemetry.proto
|
||||
// protoc-gen-go v1.34.2
|
||||
// protoc v5.29.3
|
||||
// source: telemetry.proto
|
||||
|
||||
package proto
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
|
||||
reflect "reflect"
|
||||
sync "sync"
|
||||
unsafe "unsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -23,9 +22,12 @@ const (
|
||||
|
||||
// TelemetryData represents cluster-level telemetry information
|
||||
type TelemetryData struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
// Unique cluster identifier (generated in-memory)
|
||||
TopologyId string `protobuf:"bytes,1,opt,name=topology_id,json=topologyId,proto3" json:"topology_id,omitempty"`
|
||||
ClusterId string `protobuf:"bytes,1,opt,name=cluster_id,json=clusterId,proto3" json:"cluster_id,omitempty"`
|
||||
// SeaweedFS version
|
||||
Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
|
||||
// Operating system (e.g., "linux/amd64")
|
||||
@@ -41,16 +43,16 @@ type TelemetryData struct {
|
||||
// Number of broker servers in the cluster
|
||||
BrokerCount int32 `protobuf:"varint,10,opt,name=broker_count,json=brokerCount,proto3" json:"broker_count,omitempty"`
|
||||
// Unix timestamp when the data was collected
|
||||
Timestamp int64 `protobuf:"varint,11,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
Timestamp int64 `protobuf:"varint,11,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
|
||||
}
|
||||
|
||||
func (x *TelemetryData) Reset() {
|
||||
*x = TelemetryData{}
|
||||
mi := &file_telemetry_proto_telemetry_proto_msgTypes[0]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_telemetry_proto_msgTypes[0]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *TelemetryData) String() string {
|
||||
@@ -60,8 +62,8 @@ func (x *TelemetryData) String() string {
|
||||
func (*TelemetryData) ProtoMessage() {}
|
||||
|
||||
func (x *TelemetryData) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_telemetry_proto_telemetry_proto_msgTypes[0]
|
||||
if x != nil {
|
||||
mi := &file_telemetry_proto_msgTypes[0]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
@@ -73,12 +75,12 @@ func (x *TelemetryData) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use TelemetryData.ProtoReflect.Descriptor instead.
|
||||
func (*TelemetryData) Descriptor() ([]byte, []int) {
|
||||
return file_telemetry_proto_telemetry_proto_rawDescGZIP(), []int{0}
|
||||
return file_telemetry_proto_rawDescGZIP(), []int{0}
|
||||
}
|
||||
|
||||
func (x *TelemetryData) GetTopologyId() string {
|
||||
func (x *TelemetryData) GetClusterId() string {
|
||||
if x != nil {
|
||||
return x.TopologyId
|
||||
return x.ClusterId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -141,17 +143,20 @@ func (x *TelemetryData) GetTimestamp() int64 {
|
||||
|
||||
// TelemetryRequest is sent from SeaweedFS clusters to the telemetry server
|
||||
type TelemetryRequest struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Data *TelemetryData `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Data *TelemetryData `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
|
||||
}
|
||||
|
||||
func (x *TelemetryRequest) Reset() {
|
||||
*x = TelemetryRequest{}
|
||||
mi := &file_telemetry_proto_telemetry_proto_msgTypes[1]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_telemetry_proto_msgTypes[1]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *TelemetryRequest) String() string {
|
||||
@@ -161,8 +166,8 @@ func (x *TelemetryRequest) String() string {
|
||||
func (*TelemetryRequest) ProtoMessage() {}
|
||||
|
||||
func (x *TelemetryRequest) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_telemetry_proto_telemetry_proto_msgTypes[1]
|
||||
if x != nil {
|
||||
mi := &file_telemetry_proto_msgTypes[1]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
@@ -174,7 +179,7 @@ func (x *TelemetryRequest) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use TelemetryRequest.ProtoReflect.Descriptor instead.
|
||||
func (*TelemetryRequest) Descriptor() ([]byte, []int) {
|
||||
return file_telemetry_proto_telemetry_proto_rawDescGZIP(), []int{1}
|
||||
return file_telemetry_proto_rawDescGZIP(), []int{1}
|
||||
}
|
||||
|
||||
func (x *TelemetryRequest) GetData() *TelemetryData {
|
||||
@@ -186,18 +191,21 @@ func (x *TelemetryRequest) GetData() *TelemetryData {
|
||||
|
||||
// TelemetryResponse is returned by the telemetry server
|
||||
type TelemetryResponse struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
|
||||
Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
|
||||
Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
|
||||
}
|
||||
|
||||
func (x *TelemetryResponse) Reset() {
|
||||
*x = TelemetryResponse{}
|
||||
mi := &file_telemetry_proto_telemetry_proto_msgTypes[2]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_telemetry_proto_msgTypes[2]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *TelemetryResponse) String() string {
|
||||
@@ -207,8 +215,8 @@ func (x *TelemetryResponse) String() string {
|
||||
func (*TelemetryResponse) ProtoMessage() {}
|
||||
|
||||
func (x *TelemetryResponse) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_telemetry_proto_telemetry_proto_msgTypes[2]
|
||||
if x != nil {
|
||||
mi := &file_telemetry_proto_msgTypes[2]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
@@ -220,7 +228,7 @@ func (x *TelemetryResponse) ProtoReflect() protoreflect.Message {
|
||||
|
||||
// Deprecated: Use TelemetryResponse.ProtoReflect.Descriptor instead.
|
||||
func (*TelemetryResponse) Descriptor() ([]byte, []int) {
|
||||
return file_telemetry_proto_telemetry_proto_rawDescGZIP(), []int{2}
|
||||
return file_telemetry_proto_rawDescGZIP(), []int{2}
|
||||
}
|
||||
|
||||
func (x *TelemetryResponse) GetSuccess() bool {
|
||||
@@ -237,49 +245,66 @@ func (x *TelemetryResponse) GetMessage() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
var File_telemetry_proto_telemetry_proto protoreflect.FileDescriptor
|
||||
var File_telemetry_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_telemetry_proto_telemetry_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"\x1ftelemetry/proto/telemetry.proto\x12\ttelemetry\"\xd0\x02\n" +
|
||||
"\rTelemetryData\x12\x1f\n" +
|
||||
"\vtopology_id\x18\x01 \x01(\tR\n" +
|
||||
"topologyId\x12\x18\n" +
|
||||
"\aversion\x18\x02 \x01(\tR\aversion\x12\x0e\n" +
|
||||
"\x02os\x18\x03 \x01(\tR\x02os\x12.\n" +
|
||||
"\x13volume_server_count\x18\x06 \x01(\x05R\x11volumeServerCount\x12(\n" +
|
||||
"\x10total_disk_bytes\x18\a \x01(\x04R\x0etotalDiskBytes\x12,\n" +
|
||||
"\x12total_volume_count\x18\b \x01(\x05R\x10totalVolumeCount\x12\x1f\n" +
|
||||
"\vfiler_count\x18\t \x01(\x05R\n" +
|
||||
"filerCount\x12!\n" +
|
||||
"\fbroker_count\x18\n" +
|
||||
" \x01(\x05R\vbrokerCount\x12\x1c\n" +
|
||||
"\ttimestamp\x18\v \x01(\x03R\ttimestampJ\x04\b\x04\x10\x05J\x04\b\x05\x10\x06\"@\n" +
|
||||
"\x10TelemetryRequest\x12,\n" +
|
||||
"\x04data\x18\x01 \x01(\v2\x18.telemetry.TelemetryDataR\x04data\"G\n" +
|
||||
"\x11TelemetryResponse\x12\x18\n" +
|
||||
"\asuccess\x18\x01 \x01(\bR\asuccess\x12\x18\n" +
|
||||
"\amessage\x18\x02 \x01(\tR\amessageB0Z.github.com/seaweedfs/seaweedfs/telemetry/protob\x06proto3"
|
||||
|
||||
var (
|
||||
file_telemetry_proto_telemetry_proto_rawDescOnce sync.Once
|
||||
file_telemetry_proto_telemetry_proto_rawDescData []byte
|
||||
)
|
||||
|
||||
func file_telemetry_proto_telemetry_proto_rawDescGZIP() []byte {
|
||||
file_telemetry_proto_telemetry_proto_rawDescOnce.Do(func() {
|
||||
file_telemetry_proto_telemetry_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_telemetry_proto_telemetry_proto_rawDesc), len(file_telemetry_proto_telemetry_proto_rawDesc)))
|
||||
})
|
||||
return file_telemetry_proto_telemetry_proto_rawDescData
|
||||
var file_telemetry_proto_rawDesc = []byte{
|
||||
0x0a, 0x0f, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x12, 0x09, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x22, 0xce, 0x02, 0x0a,
|
||||
0x0d, 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d,
|
||||
0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
|
||||
0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64, 0x12, 0x18, 0x0a,
|
||||
0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07,
|
||||
0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x73, 0x18, 0x03, 0x20,
|
||||
0x01, 0x28, 0x09, 0x52, 0x02, 0x6f, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x76, 0x6f, 0x6c, 0x75, 0x6d,
|
||||
0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06,
|
||||
0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x53, 0x65, 0x72, 0x76,
|
||||
0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x6f, 0x74, 0x61, 0x6c,
|
||||
0x5f, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28,
|
||||
0x04, 0x52, 0x0e, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x69, 0x73, 0x6b, 0x42, 0x79, 0x74, 0x65,
|
||||
0x73, 0x12, 0x2c, 0x0a, 0x12, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x76, 0x6f, 0x6c, 0x75, 0x6d,
|
||||
0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x74,
|
||||
0x6f, 0x74, 0x61, 0x6c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12,
|
||||
0x1f, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x09,
|
||||
0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74,
|
||||
0x12, 0x21, 0x0a, 0x0c, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74,
|
||||
0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x62, 0x72, 0x6f, 0x6b, 0x65, 0x72, 0x43, 0x6f,
|
||||
0x75, 0x6e, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70,
|
||||
0x18, 0x0b, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d,
|
||||
0x70, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x22, 0x40, 0x0a,
|
||||
0x10, 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x12, 0x2c, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32,
|
||||
0x18, 0x2e, 0x74, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x2e, 0x54, 0x65, 0x6c, 0x65,
|
||||
0x6d, 0x65, 0x74, 0x72, 0x79, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22,
|
||||
0x47, 0x0a, 0x11, 0x54, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70,
|
||||
0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x18,
|
||||
0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x30, 0x5a, 0x2e, 0x67, 0x69, 0x74, 0x68,
|
||||
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73,
|
||||
0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x74, 0x65, 0x6c, 0x65, 0x6d,
|
||||
0x65, 0x74, 0x72, 0x79, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x33,
|
||||
}
|
||||
|
||||
var file_telemetry_proto_telemetry_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
|
||||
var file_telemetry_proto_telemetry_proto_goTypes = []any{
|
||||
var (
|
||||
file_telemetry_proto_rawDescOnce sync.Once
|
||||
file_telemetry_proto_rawDescData = file_telemetry_proto_rawDesc
|
||||
)
|
||||
|
||||
func file_telemetry_proto_rawDescGZIP() []byte {
|
||||
file_telemetry_proto_rawDescOnce.Do(func() {
|
||||
file_telemetry_proto_rawDescData = protoimpl.X.CompressGZIP(file_telemetry_proto_rawDescData)
|
||||
})
|
||||
return file_telemetry_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_telemetry_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
|
||||
var file_telemetry_proto_goTypes = []any{
|
||||
(*TelemetryData)(nil), // 0: telemetry.TelemetryData
|
||||
(*TelemetryRequest)(nil), // 1: telemetry.TelemetryRequest
|
||||
(*TelemetryResponse)(nil), // 2: telemetry.TelemetryResponse
|
||||
}
|
||||
var file_telemetry_proto_telemetry_proto_depIdxs = []int32{
|
||||
var file_telemetry_proto_depIdxs = []int32{
|
||||
0, // 0: telemetry.TelemetryRequest.data:type_name -> telemetry.TelemetryData
|
||||
1, // [1:1] is the sub-list for method output_type
|
||||
1, // [1:1] is the sub-list for method input_type
|
||||
@@ -288,26 +313,65 @@ var file_telemetry_proto_telemetry_proto_depIdxs = []int32{
|
||||
0, // [0:1] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_telemetry_proto_telemetry_proto_init() }
|
||||
func file_telemetry_proto_telemetry_proto_init() {
|
||||
if File_telemetry_proto_telemetry_proto != nil {
|
||||
func init() { file_telemetry_proto_init() }
|
||||
func file_telemetry_proto_init() {
|
||||
if File_telemetry_proto != nil {
|
||||
return
|
||||
}
|
||||
if !protoimpl.UnsafeEnabled {
|
||||
file_telemetry_proto_msgTypes[0].Exporter = func(v any, i int) any {
|
||||
switch v := v.(*TelemetryData); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_telemetry_proto_msgTypes[1].Exporter = func(v any, i int) any {
|
||||
switch v := v.(*TelemetryRequest); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_telemetry_proto_msgTypes[2].Exporter = func(v any, i int) any {
|
||||
switch v := v.(*TelemetryResponse); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
File: protoimpl.DescBuilder{
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_telemetry_proto_telemetry_proto_rawDesc), len(file_telemetry_proto_telemetry_proto_rawDesc)),
|
||||
RawDescriptor: file_telemetry_proto_rawDesc,
|
||||
NumEnums: 0,
|
||||
NumMessages: 3,
|
||||
NumExtensions: 0,
|
||||
NumServices: 0,
|
||||
},
|
||||
GoTypes: file_telemetry_proto_telemetry_proto_goTypes,
|
||||
DependencyIndexes: file_telemetry_proto_telemetry_proto_depIdxs,
|
||||
MessageInfos: file_telemetry_proto_telemetry_proto_msgTypes,
|
||||
GoTypes: file_telemetry_proto_goTypes,
|
||||
DependencyIndexes: file_telemetry_proto_depIdxs,
|
||||
MessageInfos: file_telemetry_proto_msgTypes,
|
||||
}.Build()
|
||||
File_telemetry_proto_telemetry_proto = out.File
|
||||
file_telemetry_proto_telemetry_proto_goTypes = nil
|
||||
file_telemetry_proto_telemetry_proto_depIdxs = nil
|
||||
File_telemetry_proto = out.File
|
||||
file_telemetry_proto_rawDesc = nil
|
||||
file_telemetry_proto_goTypes = nil
|
||||
file_telemetry_proto_depIdxs = nil
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ option go_package = "github.com/seaweedfs/seaweedfs/telemetry/proto";
|
||||
// TelemetryData represents cluster-level telemetry information
|
||||
message TelemetryData {
|
||||
// Unique cluster identifier (generated in-memory)
|
||||
string topology_id = 1;
|
||||
string cluster_id = 1;
|
||||
|
||||
// SeaweedFS version
|
||||
string version = 2;
|
||||
|
||||
@@ -7,7 +7,7 @@ toolchain go1.25.0
|
||||
require (
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000
|
||||
google.golang.org/protobuf v1.36.11
|
||||
google.golang.org/protobuf v1.36.8
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -16,9 +16,9 @@ require (
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.19.2 // indirect
|
||||
github.com/prometheus/procfs v0.17.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/seaweedfs/seaweedfs => ../..
|
||||
|
||||
@@ -26,7 +26,6 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
|
||||
github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
|
||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
@@ -37,10 +36,8 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
||||
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
|
||||
@@ -1,187 +0,0 @@
|
||||
# Makefile for EC integration testing
|
||||
# Usage:
|
||||
# make start - Start the test cluster (master + 6 volume servers + filer)
|
||||
# make stop - Stop the test cluster
|
||||
# make populate - Populate test data (~300MB across 7 volumes)
|
||||
# make shell - Open weed shell connected to the test cluster
|
||||
# make clean - Stop cluster and remove all test data
|
||||
# make setup - Start cluster and populate data (one command)
|
||||
#
|
||||
# Requirements: curl, jq
|
||||
|
||||
WEED_BINARY := $(shell pwd)/../../weed/weed
|
||||
TEST_DIR := /tmp/ec_manual_test
|
||||
# Use non-standard ports to avoid conflicts with existing SeaweedFS servers
|
||||
MASTER_PORT := 29333
|
||||
FILER_PORT := 28888
|
||||
VOLUME_BASE_PORT := 28080
|
||||
NUM_VOLUME_SERVERS := 6
|
||||
VOLUME_SIZE_LIMIT_MB := 30
|
||||
MAX_VOLUMES_PER_SERVER := 10
|
||||
|
||||
# Build weed binary if it doesn't exist
|
||||
$(WEED_BINARY):
|
||||
cd ../../weed && go build -o weed .
|
||||
|
||||
.PHONY: build
|
||||
build: $(WEED_BINARY)
|
||||
|
||||
.PHONY: start
|
||||
start: build
|
||||
@echo "=== Starting SeaweedFS test cluster ==="
|
||||
@mkdir -p $(TEST_DIR)/master $(TEST_DIR)/filer
|
||||
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do mkdir -p $(TEST_DIR)/volume$$i; done
|
||||
@# Create security.toml with JWT disabled
|
||||
@echo "# Disable JWT for testing" > $(TEST_DIR)/security.toml
|
||||
@echo '[jwt.signing]' >> $(TEST_DIR)/security.toml
|
||||
@echo 'key = ""' >> $(TEST_DIR)/security.toml
|
||||
@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
|
||||
@echo '' >> $(TEST_DIR)/security.toml
|
||||
@echo '[jwt.signing.read]' >> $(TEST_DIR)/security.toml
|
||||
@echo 'key = ""' >> $(TEST_DIR)/security.toml
|
||||
@echo 'expires_after_seconds = 0' >> $(TEST_DIR)/security.toml
|
||||
@# Create filer.toml with leveldb2
|
||||
@echo '[leveldb2]' > $(TEST_DIR)/filer.toml
|
||||
@echo 'enabled = true' >> $(TEST_DIR)/filer.toml
|
||||
@echo 'dir = "$(TEST_DIR)/filer/filerldb2"' >> $(TEST_DIR)/filer.toml
|
||||
@# Start master
|
||||
@echo "Starting master on port $(MASTER_PORT)..."
|
||||
@cd $(TEST_DIR) && $(WEED_BINARY) master \
|
||||
-port=$(MASTER_PORT) \
|
||||
-mdir=$(TEST_DIR)/master \
|
||||
-volumeSizeLimitMB=$(VOLUME_SIZE_LIMIT_MB) \
|
||||
-ip=127.0.0.1 \
|
||||
> $(TEST_DIR)/master/master.log 2>&1 & echo $$! > $(TEST_DIR)/master.pid
|
||||
@sleep 3
|
||||
@# Start volume servers (run from TEST_DIR to find security.toml)
|
||||
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
|
||||
port=$$(($(VOLUME_BASE_PORT) + $$i)); \
|
||||
echo "Starting volume server $$i on port $$port (rack$$i)..."; \
|
||||
cd $(TEST_DIR) && $(WEED_BINARY) volume \
|
||||
-port=$$port \
|
||||
-dir=$(TEST_DIR)/volume$$i \
|
||||
-max=$(MAX_VOLUMES_PER_SERVER) \
|
||||
-master=127.0.0.1:$(MASTER_PORT) \
|
||||
-ip=127.0.0.1 \
|
||||
-dataCenter=dc1 \
|
||||
-rack=rack$$i \
|
||||
> $(TEST_DIR)/volume$$i/volume.log 2>&1 & echo $$! > $(TEST_DIR)/volume$$i.pid; \
|
||||
done
|
||||
@sleep 3
|
||||
@# Start filer (run from TEST_DIR to find security.toml)
|
||||
@echo "Starting filer on port $(FILER_PORT)..."
|
||||
@cd $(TEST_DIR) && $(WEED_BINARY) filer \
|
||||
-port=$(FILER_PORT) \
|
||||
-master=127.0.0.1:$(MASTER_PORT) \
|
||||
-ip=127.0.0.1 \
|
||||
> $(TEST_DIR)/filer/filer.log 2>&1 & echo $$! > $(TEST_DIR)/filer.pid
|
||||
@sleep 3
|
||||
@echo ""
|
||||
@echo "=== Cluster started ==="
|
||||
@echo "Master: http://127.0.0.1:$(MASTER_PORT)"
|
||||
@echo "Filer: http://127.0.0.1:$(FILER_PORT)"
|
||||
@echo "Volume servers: http://127.0.0.1:$(VOLUME_BASE_PORT) - http://127.0.0.1:$$(($(VOLUME_BASE_PORT) + $(NUM_VOLUME_SERVERS) - 1))"
|
||||
@echo ""
|
||||
@echo "Run 'make shell' to open weed shell"
|
||||
@echo "Run 'make populate' to add test data"
|
||||
|
||||
.PHONY: stop
|
||||
stop:
|
||||
@echo "=== Stopping SeaweedFS test cluster ==="
|
||||
@# Stop filer by PID
|
||||
@-[ -f $(TEST_DIR)/filer.pid ] && kill $$(cat $(TEST_DIR)/filer.pid) 2>/dev/null && rm -f $(TEST_DIR)/filer.pid || true
|
||||
@# Stop volume servers by PID
|
||||
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
|
||||
[ -f $(TEST_DIR)/volume$$i.pid ] && kill $$(cat $(TEST_DIR)/volume$$i.pid) 2>/dev/null && rm -f $(TEST_DIR)/volume$$i.pid || true; \
|
||||
done
|
||||
@# Stop master by PID
|
||||
@-[ -f $(TEST_DIR)/master.pid ] && kill $$(cat $(TEST_DIR)/master.pid) 2>/dev/null && rm -f $(TEST_DIR)/master.pid || true
|
||||
@# Fallback: use pkill with specific patterns to ensure cleanup
|
||||
@-pkill -f "weed filer.*-master=127.0.0.1:$(MASTER_PORT)" 2>/dev/null || true
|
||||
@-pkill -f "weed volume.*-dir=$(TEST_DIR)/volume" 2>/dev/null || true
|
||||
@-pkill -f "weed master.*-mdir=$(TEST_DIR)/master" 2>/dev/null || true
|
||||
@echo "Cluster stopped."
|
||||
|
||||
.PHONY: clean
|
||||
clean: stop
|
||||
@echo "Removing test data..."
|
||||
@rm -rf $(TEST_DIR)
|
||||
@echo "Clean complete."
|
||||
|
||||
.PHONY: populate
|
||||
populate:
|
||||
@echo "=== Populating test data (~300MB) ==="
|
||||
@# Create a 500KB test file template using mktemp for isolation
|
||||
@tmpfile=$$(mktemp) && \
|
||||
dd if=/dev/urandom bs=1024 count=500 of=$$tmpfile 2>/dev/null && \
|
||||
uploaded=0; \
|
||||
for i in $$(seq 1 600); do \
|
||||
response=$$(curl -s "http://127.0.0.1:$(MASTER_PORT)/dir/assign?collection=ectest&replication=000"); \
|
||||
fid=$$(echo $$response | jq -r '.fid'); \
|
||||
url=$$(echo $$response | jq -r '.url'); \
|
||||
if [ "$$fid" != "null" ] && [ -n "$$fid" ]; then \
|
||||
curl -s -F "file=@$$tmpfile;filename=file_$$i.bin" "http://$$url/$$fid" > /dev/null; \
|
||||
uploaded=$$((uploaded + 1)); \
|
||||
fi; \
|
||||
if [ $$((i % 100)) -eq 0 ]; then \
|
||||
echo "Uploaded $$uploaded files..."; \
|
||||
fi; \
|
||||
done; \
|
||||
rm -f $$tmpfile; \
|
||||
echo ""; \
|
||||
echo "=== Data population complete ==="; \
|
||||
echo "Uploaded $$uploaded files (~$$((uploaded * 500 / 1024))MB)"
|
||||
@echo ""
|
||||
@echo "Volume status:"
|
||||
@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
|
||||
'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | " Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack)"' 2>/dev/null || true
|
||||
|
||||
.PHONY: shell
|
||||
shell: build
|
||||
@echo "Opening weed shell..."
|
||||
@echo "Commands to try:"
|
||||
@echo " lock"
|
||||
@echo " volume.list"
|
||||
@echo " ec.encode -collection=ectest -quietFor=1s -force"
|
||||
@echo " ec.balance -collection=ectest"
|
||||
@echo " unlock"
|
||||
@echo ""
|
||||
@$(WEED_BINARY) shell -master=127.0.0.1:$(MASTER_PORT) -filer=127.0.0.1:$(FILER_PORT)
|
||||
|
||||
.PHONY: setup
|
||||
setup: clean start
|
||||
@sleep 2
|
||||
@$(MAKE) populate
|
||||
|
||||
.PHONY: status
|
||||
status:
|
||||
@echo "=== Cluster Status ==="
|
||||
@curl -s "http://127.0.0.1:$(MASTER_PORT)/vol/status" | jq -r \
|
||||
'.Volumes.DataCenters.dc1 | to_entries[] | .key as $$rack | .value | to_entries[] | select(.value != null) | .key as $$server | .value[] | select(.Collection == "ectest") | "Volume \(.Id): \(.FileCount) files, \((.Size/1048576*10|floor)/10)MB - \($$rack) (\($$server))"' 2>/dev/null | sort -t: -k1 -n || echo "Cluster not running"
|
||||
@echo ""
|
||||
@echo "=== EC Shards ==="
|
||||
@for i in $$(seq 0 $$(($(NUM_VOLUME_SERVERS)-1))); do \
|
||||
count=$$(ls $(TEST_DIR)/volume$$i/*.ec[0-9]* 2>/dev/null | wc -l | tr -d ' '); \
|
||||
if [ "$$count" != "0" ]; then \
|
||||
echo " volume$$i (port $$(($(VOLUME_BASE_PORT) + $$i))): $$count EC shard files"; \
|
||||
fi; \
|
||||
done
|
||||
|
||||
.PHONY: help
|
||||
help:
|
||||
@echo "EC Integration Test Makefile"
|
||||
@echo ""
|
||||
@echo "Targets:"
|
||||
@echo " make start - Start test cluster (master + 6 volume servers + filer)"
|
||||
@echo " make stop - Stop test cluster"
|
||||
@echo " make populate - Populate ~300MB of test data"
|
||||
@echo " make shell - Open weed shell"
|
||||
@echo " make setup - Clean, start, and populate (all-in-one)"
|
||||
@echo " make status - Show cluster and EC shard status"
|
||||
@echo " make clean - Stop cluster and remove all test data"
|
||||
@echo " make help - Show this help"
|
||||
@echo ""
|
||||
@echo "Quick start:"
|
||||
@echo " make setup # Start cluster and populate data"
|
||||
@echo " make shell # Open shell to run EC commands"
|
||||
|
||||
@@ -78,43 +78,6 @@ go test -v -run TestECEncodingMasterTimingRaceCondition
|
||||
go test -v -short
|
||||
```
|
||||
|
||||
## Manual Testing with Makefile
|
||||
|
||||
A Makefile is provided for manual EC testing.
|
||||
|
||||
**Requirements:** `curl`, `jq` (command-line JSON processor)
|
||||
|
||||
```bash
|
||||
# Quick start: start cluster and populate data
|
||||
make setup
|
||||
|
||||
# Open weed shell to run EC commands
|
||||
make shell
|
||||
|
||||
# Individual targets
|
||||
make start # Start test cluster (master + 6 volume servers + filer)
|
||||
make stop # Stop test cluster
|
||||
make populate # Populate ~300MB of test data
|
||||
make status # Show cluster and EC shard status
|
||||
make clean # Stop cluster and remove all test data
|
||||
make help # Show all targets
|
||||
```
|
||||
|
||||
### EC Rebalance Limited Slots (Unit Test)
|
||||
|
||||
The "no free ec shard slots" issue is tested with a **unit test** that works directly on
|
||||
topology data structures without requiring a running cluster.
|
||||
|
||||
**Location**: `weed/shell/ec_rebalance_slots_test.go`
|
||||
|
||||
Tests included:
|
||||
- `TestECRebalanceWithLimitedSlots`: Tests a topology with 6 servers, 7 EC volumes (98 shards)
|
||||
- `TestECRebalanceZeroFreeSlots`: Reproduces the exact 0 free slots scenario
|
||||
|
||||
**Known Issue**: When volume servers are at capacity (`volumeCount == maxVolumeCount`),
|
||||
the rebalance step fails with "no free ec shard slots" instead of recognizing that
|
||||
moving shards frees slots on source servers.
|
||||
|
||||
## Test Results
|
||||
|
||||
**With the fix**: Shows "Collecting volume locations for N volumes before EC encoding..." message
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
.PHONY: test clean
|
||||
|
||||
test:
|
||||
go test -v .
|
||||
|
||||
clean:
|
||||
rm -rf tmp
|
||||
@@ -1,404 +0,0 @@
|
||||
package admin_dockertest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
AdminUrl = "http://localhost:23646"
|
||||
MasterUrl = "http://localhost:9333"
|
||||
FilerUrl = "http://localhost:8888"
|
||||
)
|
||||
|
||||
// Helper to run commands in background and track PIDs for cleanup
|
||||
var runningCmds []*exec.Cmd
|
||||
|
||||
func cleanup() {
|
||||
for _, cmd := range runningCmds {
|
||||
if cmd.Process != nil {
|
||||
cmd.Process.Kill()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func startWeed(t *testing.T, name string, args ...string) *exec.Cmd {
|
||||
cmd := exec.Command("./weed_bin", args...)
|
||||
|
||||
// Create logs dir in local ./tmp
|
||||
wd, _ := os.Getwd()
|
||||
logDir := filepath.Join(wd, "tmp", "logs")
|
||||
os.MkdirAll(logDir, 0755)
|
||||
|
||||
logFile, err := os.Create(filepath.Join(logDir, name+".log"))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create log file: %v", err)
|
||||
}
|
||||
|
||||
cmd.Stdout = logFile
|
||||
cmd.Stderr = logFile
|
||||
// Set Cwd to test directory so it finds local ./tmp
|
||||
cmd.Dir = wd
|
||||
|
||||
// assume "weed_bin" binary is in project root.
|
||||
rootDir := filepath.Dir(filepath.Dir(filepath.Dir(wd)))
|
||||
cmd.Path = filepath.Join(rootDir, "weed_bin")
|
||||
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to start weed %v: %v", args, err)
|
||||
}
|
||||
runningCmds = append(runningCmds, cmd)
|
||||
return cmd
|
||||
}
|
||||
|
||||
func stopWeed(t *testing.T, cmd *exec.Cmd) {
|
||||
if cmd != nil && cmd.Process != nil {
|
||||
t.Logf("Stopping process %d", cmd.Process.Pid)
|
||||
cmd.Process.Kill()
|
||||
cmd.Wait()
|
||||
|
||||
// Remove from runningCmds to avoid double kill in cleanup
|
||||
for i, c := range runningCmds {
|
||||
if c == cmd {
|
||||
runningCmds = append(runningCmds[:i], runningCmds[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ensureEnvironment(t *testing.T) {
|
||||
// 1. Build weed binary
|
||||
wd, _ := os.Getwd()
|
||||
rootDir := filepath.Dir(filepath.Dir(filepath.Dir(wd))) // Up 3 levels
|
||||
|
||||
buildCmd := exec.Command("go", "build", "-o", "weed_bin", "./weed")
|
||||
buildCmd.Dir = rootDir
|
||||
buildCmd.Stdout = os.Stdout
|
||||
buildCmd.Stderr = os.Stderr
|
||||
if err := buildCmd.Run(); err != nil {
|
||||
t.Fatalf("Failed to build weed: %v", err)
|
||||
}
|
||||
t.Log("Successfully built weed binary")
|
||||
|
||||
// 2. Start Master
|
||||
// Use local ./tmp/master
|
||||
os.RemoveAll("tmp")
|
||||
err := os.MkdirAll(filepath.Join("tmp", "master"), 0755)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create tmp dir: %v", err)
|
||||
}
|
||||
|
||||
startWeed(t, "master", "master", "-mdir=./tmp/master", "-port=9333", "-ip=localhost", "-peers=none", "-volumeSizeLimitMB=100")
|
||||
|
||||
// Wait for master
|
||||
waitForUrl(t, MasterUrl+"/cluster/status", 10)
|
||||
|
||||
// 3. Start Volume Server (Worker)
|
||||
// Start 14 volume servers to verify RS(10,4) default EC
|
||||
for i := 1; i <= 14; i++ {
|
||||
volName := fmt.Sprintf("volume%d", i)
|
||||
port := 8080 + i - 1
|
||||
dir := filepath.Join("tmp", volName)
|
||||
os.MkdirAll(dir, 0755)
|
||||
startWeed(t, volName, "volume", "-dir="+dir, "-mserver=localhost:9333", fmt.Sprintf("-port=%d", port), "-ip=localhost")
|
||||
}
|
||||
|
||||
// 4. Start Filer
|
||||
os.MkdirAll(filepath.Join("tmp", "filer"), 0755)
|
||||
startWeed(t, "filer", "filer", "-defaultStoreDir=./tmp/filer", "-master=localhost:9333", "-port=8888", "-ip=localhost")
|
||||
waitForUrl(t, FilerUrl+"/", 60)
|
||||
|
||||
// 5. Start Workers (Maintenance)
|
||||
// We need workers to execute EC tasks
|
||||
for i := 1; i <= 2; i++ {
|
||||
workerName := fmt.Sprintf("worker%d", i)
|
||||
metricsPort := 9327 + i - 1
|
||||
debugPort := 6060 + i
|
||||
dir, _ := filepath.Abs(filepath.Join("tmp", workerName))
|
||||
os.MkdirAll(dir, 0755)
|
||||
startWeed(t, workerName, "worker", "-admin=localhost:23646", "-workingDir="+dir, fmt.Sprintf("-metricsPort=%d", metricsPort), fmt.Sprintf("-debug.port=%d", debugPort))
|
||||
}
|
||||
|
||||
// 6. Start Admin
|
||||
os.RemoveAll(filepath.Join("tmp", "admin"))
|
||||
os.MkdirAll(filepath.Join("tmp", "admin"), 0755)
|
||||
startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin")
|
||||
waitForUrl(t, AdminUrl+"/health", 60)
|
||||
|
||||
t.Log("Environment started successfully")
|
||||
}
|
||||
|
||||
func waitForUrl(t *testing.T, url string, retries int) {
|
||||
for i := 0; i < retries; i++ {
|
||||
resp, err := http.Get(url)
|
||||
if err == nil && resp.StatusCode == 200 {
|
||||
resp.Body.Close()
|
||||
return
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
t.Fatalf("Timeout waiting for %s", url)
|
||||
}
|
||||
|
||||
func TestEcEndToEnd(t *testing.T) {
|
||||
defer cleanup()
|
||||
ensureEnvironment(t)
|
||||
|
||||
client := &http.Client{}
|
||||
|
||||
// 1. Configure Global Maintenance (Scan Interval = 1s) via API
|
||||
t.Log("Configuring Global Maintenance via API...")
|
||||
|
||||
// 1.1 Fetch current config
|
||||
req, _ := http.NewRequest("GET", AdminUrl+"/api/maintenance/config", nil)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get global config: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("Failed to get global config (status %d): %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var globalConfig map[string]interface{}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&globalConfig); err != nil {
|
||||
t.Fatalf("Failed to decode global config: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// 1.2 Modify config
|
||||
globalConfig["enabled"] = true
|
||||
globalConfig["scan_interval_seconds"] = 1
|
||||
|
||||
// Ensure policy structure exists
|
||||
if globalConfig["policy"] == nil {
|
||||
globalConfig["policy"] = map[string]interface{}{}
|
||||
}
|
||||
policy, _ := globalConfig["policy"].(map[string]interface{})
|
||||
|
||||
// Ensure task_policies structure exists
|
||||
if policy["task_policies"] == nil {
|
||||
policy["task_policies"] = map[string]interface{}{}
|
||||
}
|
||||
taskPolicies, _ := policy["task_policies"].(map[string]interface{})
|
||||
|
||||
// Disable balance tasks to avoid interference with EC test
|
||||
if taskPolicies["balance"] == nil {
|
||||
taskPolicies["balance"] = map[string]interface{}{}
|
||||
}
|
||||
balancePolicy, _ := taskPolicies["balance"].(map[string]interface{})
|
||||
balancePolicy["enabled"] = false
|
||||
|
||||
// Set global max concurrent
|
||||
policy["global_max_concurrent"] = 4
|
||||
globalConfig["policy"] = policy
|
||||
|
||||
// Explicitly set required fields
|
||||
requiredFields := map[string]float64{
|
||||
"worker_timeout_seconds": 300,
|
||||
"task_timeout_seconds": 7200,
|
||||
"retry_delay_seconds": 900,
|
||||
"cleanup_interval_seconds": 86400,
|
||||
"task_retention_seconds": 604800,
|
||||
"max_retries": 3,
|
||||
}
|
||||
for field, val := range requiredFields {
|
||||
if _, ok := globalConfig[field]; !ok || globalConfig[field] == 0 {
|
||||
globalConfig[field] = val
|
||||
}
|
||||
}
|
||||
|
||||
// 1.3 Update config
|
||||
jsonBody, _ := json.Marshal(globalConfig)
|
||||
req, _ = http.NewRequest("PUT", AdminUrl+"/api/maintenance/config", bytes.NewBuffer(jsonBody))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err = client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to update global config: %v", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("Failed to update global config (status %d): %s", resp.StatusCode, string(body))
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
// 2. Configure EC Task (Short intervals) via Form API
|
||||
t.Log("Configuring EC Task via Form API...")
|
||||
formData := url.Values{}
|
||||
formData.Set("enabled", "true")
|
||||
formData.Set("scan_interval_seconds", "1")
|
||||
formData.Set("repeat_interval_seconds", "1")
|
||||
formData.Set("check_interval_seconds", "1")
|
||||
formData.Set("max_concurrent", "4")
|
||||
formData.Set("quiet_for_seconds_value", "1")
|
||||
formData.Set("quiet_for_seconds_unit", "seconds")
|
||||
formData.Set("min_size_mb", "1")
|
||||
formData.Set("fullness_ratio", "0.0001")
|
||||
|
||||
req, _ = http.NewRequest("POST", AdminUrl+"/maintenance/config/erasure_coding", strings.NewReader(formData.Encode()))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
resp, err = client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to update EC config: %v", err)
|
||||
}
|
||||
if resp.StatusCode != 200 && resp.StatusCode != 303 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("Failed to update EC config (status %d): %s", resp.StatusCode, string(body))
|
||||
}
|
||||
resp.Body.Close()
|
||||
t.Log("EC Task Configuration updated")
|
||||
|
||||
// 3. Restart Admin to pick up Global Config (Scan Interval)
|
||||
if len(runningCmds) > 0 {
|
||||
adminCmd := runningCmds[len(runningCmds)-1]
|
||||
t.Log("Restarting Admin Server to apply configuration...")
|
||||
stopWeed(t, adminCmd)
|
||||
time.Sleep(10 * time.Second)
|
||||
startWeed(t, "admin_restarted", "admin", "-master=localhost:9333", "-port=23646", "-port.grpc=33646", "-dataDir=./tmp/admin")
|
||||
waitForUrl(t, AdminUrl+"/health", 60)
|
||||
}
|
||||
|
||||
// 4. Upload a file
|
||||
fileSize := 5 * 1024 * 1024
|
||||
data := make([]byte, fileSize)
|
||||
rand.Read(data)
|
||||
fileName := fmt.Sprintf("ec_test_file_%d", time.Now().Unix())
|
||||
t.Logf("Uploading %d bytes file %s to Filer...", fileSize, fileName)
|
||||
uploadUrl := FilerUrl + "/" + fileName
|
||||
|
||||
var uploadErr error
|
||||
for i := 0; i < 10; i++ {
|
||||
req, _ := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(data))
|
||||
resp, err := client.Do(req)
|
||||
if err == nil {
|
||||
if resp.StatusCode == 201 {
|
||||
resp.Body.Close()
|
||||
uploadErr = nil
|
||||
break
|
||||
}
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
uploadErr = fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
|
||||
} else {
|
||||
uploadErr = err
|
||||
}
|
||||
t.Logf("Upload attempt %d failed: %v", i+1, uploadErr)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
|
||||
if uploadErr != nil {
|
||||
t.Fatalf("Failed to upload file after retries: %v", uploadErr)
|
||||
}
|
||||
t.Log("Upload successful")
|
||||
|
||||
// 5. Verify EC Encoding
|
||||
t.Log("Waiting for EC encoding (checking Master topology)...")
|
||||
startTime := time.Now()
|
||||
ecVerified := false
|
||||
var lastBody []byte
|
||||
|
||||
for time.Since(startTime) < 300*time.Second {
|
||||
// 5.1 Check Master Topology
|
||||
resp, err := http.Get(MasterUrl + "/dir/status")
|
||||
if err == nil {
|
||||
lastBody, _ = ioutil.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
|
||||
// Check total EC shards
|
||||
reShards := regexp.MustCompile(`"EcShards":\s*(\d+)`)
|
||||
matches := reShards.FindAllSubmatch(lastBody, -1)
|
||||
totalShards := 0
|
||||
for _, m := range matches {
|
||||
var count int
|
||||
fmt.Sscanf(string(m[1]), "%d", &count)
|
||||
totalShards += count
|
||||
}
|
||||
|
||||
if totalShards > 0 {
|
||||
t.Logf("EC encoding verified (found %d total EcShards in topology) after %d seconds", totalShards, int(time.Since(startTime).Seconds()))
|
||||
ecVerified = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// 5.2 Debug: Check workers and tasks
|
||||
wResp, wErr := http.Get(AdminUrl + "/api/maintenance/workers")
|
||||
workerCount := 0
|
||||
if wErr == nil {
|
||||
var workers []interface{}
|
||||
json.NewDecoder(wResp.Body).Decode(&workers)
|
||||
wResp.Body.Close()
|
||||
workerCount = len(workers)
|
||||
}
|
||||
|
||||
tResp, tErr := http.Get(AdminUrl + "/api/maintenance/tasks")
|
||||
taskCount := 0
|
||||
if tErr == nil {
|
||||
var tasks []interface{}
|
||||
json.NewDecoder(tResp.Body).Decode(&tasks)
|
||||
tResp.Body.Close()
|
||||
taskCount = len(tasks)
|
||||
}
|
||||
t.Logf("Waiting for EC... (Workers: %d, Active Tasks: %d)", workerCount, taskCount)
|
||||
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
|
||||
if !ecVerified {
|
||||
dumpLogs(t)
|
||||
t.Fatalf("Timed out waiting for EC encoding verified in Topology. Last body: %s", string(lastBody))
|
||||
}
|
||||
|
||||
// 6. Verification: Read back the file
|
||||
t.Log("Reading back file...")
|
||||
resp, err = http.Get(uploadUrl)
|
||||
if err != nil {
|
||||
dumpLogs(t)
|
||||
t.Fatalf("Failed to read back file: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
dumpLogs(t)
|
||||
t.Fatalf("Read back failed status: %d", resp.StatusCode)
|
||||
}
|
||||
content, _ := io.ReadAll(resp.Body)
|
||||
if len(content) != fileSize {
|
||||
dumpLogs(t)
|
||||
t.Fatalf("Read back size mismatch: got %d, want %d", len(content), fileSize)
|
||||
}
|
||||
|
||||
// Verify byte-wise content equality
|
||||
if !bytes.Equal(content, data) {
|
||||
dumpLogs(t)
|
||||
t.Fatalf("Read back content mismatch: uploaded and downloaded data differ")
|
||||
}
|
||||
|
||||
t.Log("Test PASS: EC encoding and read back successful!")
|
||||
}
|
||||
|
||||
func dumpLogs(t *testing.T) {
|
||||
wd, _ := os.Getwd()
|
||||
logDir := filepath.Join(wd, "tmp", "logs")
|
||||
files, _ := os.ReadDir(logDir)
|
||||
for _, f := range files {
|
||||
if strings.HasSuffix(f.Name(), ".log") {
|
||||
content, _ := os.ReadFile(filepath.Join(logDir, f.Name()))
|
||||
t.Logf("--- LOG DUMP: %s ---\n%s\n--- END LOG ---", f.Name(), string(content))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -15,12 +15,8 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/shell"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
@@ -35,8 +31,9 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
|
||||
}
|
||||
|
||||
// Create temporary directory for test data
|
||||
// Using t.TempDir() automatically preserves logs when tests fail
|
||||
testDir := t.TempDir()
|
||||
testDir, err := os.MkdirTemp("", "seaweedfs_ec_integration_test_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(testDir)
|
||||
|
||||
// Start SeaweedFS cluster with multiple volume servers
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
@@ -99,12 +96,13 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) {
|
||||
// This simulates the race condition where EC encoding updates master metadata
|
||||
// but volume location collection happens after that update
|
||||
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// First acquire the lock (required for EC encode)
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err = lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Execute EC encoding - test the timing directly
|
||||
var encodeOutput bytes.Buffer
|
||||
@@ -245,8 +243,9 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) {
|
||||
}
|
||||
|
||||
// Create temporary directory for test data
|
||||
// Using t.TempDir() automatically preserves logs when tests fail
|
||||
testDir := t.TempDir()
|
||||
testDir, err := os.MkdirTemp("", "seaweedfs_ec_race_test_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(testDir)
|
||||
|
||||
// Start SeaweedFS cluster
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
@@ -388,7 +387,7 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
// Find weed binary
|
||||
weedBinary := findWeedBinary()
|
||||
if weedBinary == "" {
|
||||
return nil, fmt.Errorf("weed binary not found - build with 'go build' or 'make' first")
|
||||
return nil, fmt.Errorf("weed binary not found")
|
||||
}
|
||||
|
||||
cluster := &TestCluster{}
|
||||
@@ -397,11 +396,6 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
masterDir := filepath.Join(dataDir, "master")
|
||||
os.MkdirAll(masterDir, 0755)
|
||||
|
||||
// Create an empty security.toml to disable JWT authentication in tests
|
||||
// This prevents the test from picking up ~/.seaweedfs/security.toml
|
||||
securityToml := filepath.Join(dataDir, "security.toml")
|
||||
os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644)
|
||||
|
||||
// Start master server
|
||||
masterCmd := exec.CommandContext(ctx, weedBinary, "master",
|
||||
"-port", "9333",
|
||||
@@ -410,7 +404,6 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
"-ip", "127.0.0.1",
|
||||
"-peers", "none", // Faster startup when no multiple masters needed
|
||||
)
|
||||
masterCmd.Dir = dataDir // Run from test dir so it picks up our security.toml
|
||||
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
if err != nil {
|
||||
@@ -443,7 +436,6 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e
|
||||
"-dataCenter", "dc1",
|
||||
"-rack", rack,
|
||||
)
|
||||
volumeCmd.Dir = dataDir // Run from test dir so it picks up our security.toml
|
||||
|
||||
volumeLogFile, err := os.Create(filepath.Join(volumeDir, "volume.log"))
|
||||
if err != nil {
|
||||
@@ -478,10 +470,6 @@ func findWeedBinary() string {
|
||||
|
||||
for _, candidate := range candidates {
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
// Convert to absolute path so it works when command's Dir is changed
|
||||
if absPath, err := filepath.Abs(candidate); err == nil {
|
||||
return absPath
|
||||
}
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
@@ -663,55 +651,6 @@ type commandRunner interface {
|
||||
Do([]string, *shell.CommandEnv, io.Writer) error
|
||||
}
|
||||
|
||||
// tryLockWithTimeout attempts to acquire the shell lock with a timeout.
|
||||
// Returns true if lock was acquired, false if timeout or error occurred.
|
||||
// If lock was acquired, the caller must call the returned unlock function.
|
||||
func tryLockWithTimeout(t *testing.T, commandEnv *shell.CommandEnv, timeout time.Duration) (locked bool, unlock func()) {
|
||||
t.Helper()
|
||||
|
||||
type lockResult struct {
|
||||
err error
|
||||
output string
|
||||
}
|
||||
|
||||
lockDone := make(chan lockResult, 1)
|
||||
go func() {
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
lockDone <- lockResult{err: err, output: lockOutput.String()}
|
||||
}()
|
||||
|
||||
select {
|
||||
case res := <-lockDone:
|
||||
if res.err != nil {
|
||||
t.Logf("lock command failed: %v, output: %s", res.err, res.output)
|
||||
return false, nil
|
||||
}
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
return true, func() {
|
||||
var unlockOutput bytes.Buffer
|
||||
_ = unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
}
|
||||
case <-time.After(timeout):
|
||||
t.Logf("timed out acquiring lock after %s", timeout)
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
|
||||
// connectToMasterAndSync connects the command environment to the master and waits for sync.
|
||||
// This helper reduces code duplication across test functions.
|
||||
func connectToMasterAndSync(ctx context.Context, t *testing.T, commandEnv *shell.CommandEnv) {
|
||||
t.Helper()
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
// Wait for master client to fully sync
|
||||
t.Log("Waiting for master client to sync...")
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
|
||||
// captureCommandOutput executes a shell command and captures its output from both
|
||||
// stdout/stderr and the command's buffer. This reduces code duplication in tests.
|
||||
func captureCommandOutput(t *testing.T, cmd commandRunner, args []string, commandEnv *shell.CommandEnv) (output string, err error) {
|
||||
@@ -722,16 +661,17 @@ func captureCommandOutput(t *testing.T, cmd commandRunner, args []string, comman
|
||||
r, w, pipeErr := os.Pipe()
|
||||
require.NoError(t, pipeErr)
|
||||
|
||||
defer func() {
|
||||
_ = w.Close()
|
||||
os.Stdout = oldStdout
|
||||
os.Stderr = oldStderr
|
||||
}()
|
||||
|
||||
os.Stdout = w
|
||||
os.Stderr = w
|
||||
|
||||
cmdErr := cmd.Do(args, commandEnv, &outBuf)
|
||||
|
||||
// Close write end BEFORE reading to signal EOF to the reader
|
||||
_ = w.Close()
|
||||
os.Stdout = oldStdout
|
||||
os.Stderr = oldStderr
|
||||
|
||||
capturedOutput, readErr := io.ReadAll(r)
|
||||
_ = r.Close()
|
||||
require.NoError(t, readErr)
|
||||
@@ -783,8 +723,9 @@ func TestDiskAwareECRebalancing(t *testing.T) {
|
||||
t.Skip("Skipping disk-aware integration test in short mode")
|
||||
}
|
||||
|
||||
// Using t.TempDir() automatically preserves logs when tests fail
|
||||
testDir := t.TempDir()
|
||||
testDir, err := os.MkdirTemp("", "seaweedfs_disk_aware_ec_test_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(testDir)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||||
defer cancel()
|
||||
@@ -812,7 +753,12 @@ func TestDiskAwareECRebalancing(t *testing.T) {
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
connectToMasterAndSync(ctx, t, commandEnv)
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
// Wait for master client to fully sync
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// Upload test data to create a volume - retry if volumes not ready
|
||||
var volumeId needle.VolumeId
|
||||
@@ -1216,8 +1162,9 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
t.Skip("Skipping disk type integration test in short mode")
|
||||
}
|
||||
|
||||
// Using t.TempDir() automatically preserves logs when tests fail
|
||||
testDir := t.TempDir()
|
||||
testDir, err := os.MkdirTemp("", "seaweedfs_ec_disktype_test_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(testDir)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||||
defer cancel()
|
||||
@@ -1245,7 +1192,12 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
connectToMasterAndSync(ctx, t, commandEnv)
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
// Wait for master client to fully sync
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// Upload test data to create a volume - retry if volumes not ready
|
||||
var volumeId needle.VolumeId
|
||||
@@ -1272,12 +1224,19 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_encode_with_ssd_disktype", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock first
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Defer unlock to ensure it's always released
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// Execute EC encoding with SSD disk type
|
||||
ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
|
||||
@@ -1307,12 +1266,19 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_balance_with_ssd_disktype", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock first
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Defer unlock to ensure it's always released
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// Execute EC balance with SSD disk type
|
||||
ecBalanceCmd := shell.Commands[findCommandIndex("ec.balance")]
|
||||
@@ -1355,12 +1321,19 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_encode_with_source_disktype", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Test that -sourceDiskType flag is accepted
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Defer unlock to ensure it's always released
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// Execute EC encoding with sourceDiskType filter
|
||||
ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
|
||||
@@ -1389,12 +1362,19 @@ func TestECDiskTypeSupport(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_decode_with_disktype", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Test that ec.decode accepts -diskType flag
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Defer unlock to ensure it's always released
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// Execute EC decode with disk type
|
||||
ecDecodeCmd := shell.Commands[findCommandIndex("ec.decode")]
|
||||
@@ -1440,7 +1420,6 @@ func startClusterWithDiskType(ctx context.Context, dataDir string, diskType stri
|
||||
"-mdir", masterDir,
|
||||
"-volumeSizeLimitMB", "10",
|
||||
"-ip", "127.0.0.1",
|
||||
"-peers", "none",
|
||||
)
|
||||
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
@@ -1556,8 +1535,9 @@ func TestECDiskTypeMixedCluster(t *testing.T) {
|
||||
t.Skip("Skipping mixed disk type integration test in short mode")
|
||||
}
|
||||
|
||||
// Using t.TempDir() automatically preserves logs when tests fail
|
||||
testDir := t.TempDir()
|
||||
testDir, err := os.MkdirTemp("", "seaweedfs_ec_mixed_disktype_test_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(testDir)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||||
defer cancel()
|
||||
@@ -1585,7 +1565,12 @@ func TestECDiskTypeMixedCluster(t *testing.T) {
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
connectToMasterAndSync(ctx, t, commandEnv)
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
// Wait for master client to fully sync
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
t.Run("upload_to_ssd_and_hdd", func(t *testing.T) {
|
||||
// Upload to SSD
|
||||
@@ -1620,12 +1605,19 @@ func TestECDiskTypeMixedCluster(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_balance_targets_correct_disk_type", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock first
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Defer unlock to ensure it's always released
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// Run ec.balance for SSD collection with -diskType=ssd
|
||||
var ssdOutput bytes.Buffer
|
||||
@@ -1671,7 +1663,6 @@ func startMixedDiskTypeCluster(ctx context.Context, dataDir string) (*MultiDiskC
|
||||
"-mdir", masterDir,
|
||||
"-volumeSizeLimitMB", "10",
|
||||
"-ip", "127.0.0.1",
|
||||
"-peers", "none",
|
||||
)
|
||||
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
@@ -1745,8 +1736,9 @@ func TestEvacuationFallbackBehavior(t *testing.T) {
|
||||
t.Skip("Skipping evacuation fallback test in short mode")
|
||||
}
|
||||
|
||||
// Using t.TempDir() automatically preserves logs when tests fail
|
||||
testDir := t.TempDir()
|
||||
testDir, err := os.MkdirTemp("", "seaweedfs_evacuation_fallback_test_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(testDir)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||||
defer cancel()
|
||||
@@ -1772,7 +1764,11 @@ func TestEvacuationFallbackBehavior(t *testing.T) {
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
connectToMasterAndSync(ctx, t, commandEnv)
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
t.Run("fallback_when_same_disktype_full", func(t *testing.T) {
|
||||
// This test verifies that when evacuating SSD EC shards from a server,
|
||||
@@ -1797,12 +1793,18 @@ func TestEvacuationFallbackBehavior(t *testing.T) {
|
||||
|
||||
time.Sleep(3 * time.Second)
|
||||
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// EC encode the SSD volume
|
||||
var encodeOutput bytes.Buffer
|
||||
@@ -1838,8 +1840,9 @@ func TestCrossRackECPlacement(t *testing.T) {
|
||||
t.Skip("Skipping cross-rack EC placement test in short mode")
|
||||
}
|
||||
|
||||
// Using t.TempDir() automatically preserves logs when tests fail
|
||||
testDir := t.TempDir()
|
||||
testDir, err := os.MkdirTemp("", "seaweedfs_cross_rack_ec_test_")
|
||||
require.NoError(t, err)
|
||||
defer os.RemoveAll(testDir)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||||
defer cancel()
|
||||
@@ -1865,7 +1868,11 @@ func TestCrossRackECPlacement(t *testing.T) {
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
connectToMasterAndSync(ctx, t, commandEnv)
|
||||
// Connect to master - use the main test context to avoid early disconnection
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx)
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// Upload test data
|
||||
testData := []byte("Cross-rack EC placement test data - needs to be distributed across racks")
|
||||
@@ -1884,12 +1891,18 @@ func TestCrossRackECPlacement(t *testing.T) {
|
||||
time.Sleep(3 * time.Second)
|
||||
|
||||
t.Run("ec_encode_cross_rack", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
// EC encode with rack-aware placement
|
||||
// Note: uploadTestDataToMaster uses collection "test" by default
|
||||
@@ -1934,12 +1947,18 @@ func TestCrossRackECPlacement(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("ec_balance_respects_rack_placement", func(t *testing.T) {
|
||||
// Try to get lock with timeout to avoid hanging
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout - master may not be ready")
|
||||
// Get lock
|
||||
lockCmd := shell.Commands[findCommandIndex("lock")]
|
||||
var lockOutput bytes.Buffer
|
||||
err := lockCmd.Do([]string{}, commandEnv, &lockOutput)
|
||||
if err != nil {
|
||||
t.Logf("Lock command failed: %v", err)
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
unlockCmd := shell.Commands[findCommandIndex("unlock")]
|
||||
var unlockOutput bytes.Buffer
|
||||
defer unlockCmd.Do([]string{}, commandEnv, &unlockOutput)
|
||||
|
||||
initialDistribution := countShardsPerRack(testDir, uint32(volumeId))
|
||||
t.Logf("Initial rack distribution: %v", initialDistribution)
|
||||
@@ -1987,7 +2006,6 @@ func startLimitedSsdCluster(ctx context.Context, dataDir string) (*MultiDiskClus
|
||||
"-mdir", masterDir,
|
||||
"-volumeSizeLimitMB", "10",
|
||||
"-ip", "127.0.0.1",
|
||||
"-peers", "none",
|
||||
)
|
||||
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
@@ -2078,7 +2096,6 @@ func startMultiRackCluster(ctx context.Context, dataDir string) (*MultiDiskClust
|
||||
"-mdir", masterDir,
|
||||
"-volumeSizeLimitMB", "10",
|
||||
"-ip", "127.0.0.1",
|
||||
"-peers", "none",
|
||||
)
|
||||
|
||||
masterLogFile, err := os.Create(filepath.Join(masterDir, "master.log"))
|
||||
@@ -2182,279 +2199,3 @@ func countShardsPerRack(testDir string, volumeId uint32) map[string]int {
|
||||
|
||||
return rackDistribution
|
||||
}
|
||||
|
||||
// TestECEncodeReplicatedVolumeSync tests that ec.encode properly syncs missing entries
|
||||
// between replicas before encoding. This addresses issue #7797.
|
||||
func TestECEncodeReplicatedVolumeSync(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping replicated volume sync integration test in short mode")
|
||||
}
|
||||
|
||||
// Create temporary directory for test data
|
||||
// Using t.TempDir() automatically preserves logs when tests fail
|
||||
testDir := t.TempDir()
|
||||
|
||||
// Start SeaweedFS cluster
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cluster, err := startSeaweedFSCluster(ctx, testDir)
|
||||
require.NoError(t, err)
|
||||
defer cluster.Stop()
|
||||
|
||||
// Wait for servers to be ready
|
||||
require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second))
|
||||
for i := 0; i < 6; i++ {
|
||||
require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second))
|
||||
}
|
||||
|
||||
// Create command environment
|
||||
options := &shell.ShellOptions{
|
||||
Masters: stringPtr("127.0.0.1:9333"),
|
||||
GrpcDialOption: grpc.WithInsecure(),
|
||||
FilerGroup: stringPtr("default"),
|
||||
}
|
||||
commandEnv := shell.NewCommandEnv(options)
|
||||
|
||||
// Connect to master with longer timeout
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel2()
|
||||
go commandEnv.MasterClient.KeepConnectedToMaster(ctx2)
|
||||
commandEnv.MasterClient.WaitUntilConnected(ctx2)
|
||||
|
||||
// Wait for volume servers to register with master
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// Test: Create replicated volume and verify sync behavior with consistent replicas
|
||||
t.Run("sync_replicated_volume_consistent", func(t *testing.T) {
|
||||
const testCollection = "replicated_consistent"
|
||||
const testReplication = "010" // 2 copies on different servers on the same rack
|
||||
|
||||
// Retry a few times as volume servers may still be registering
|
||||
var volumeId needle.VolumeId
|
||||
var uploadErr error
|
||||
for retry := 0; retry < 5; retry++ {
|
||||
volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
|
||||
if uploadErr == nil {
|
||||
break
|
||||
}
|
||||
t.Logf("Upload attempt %d failed: %v, retrying...", retry+1, uploadErr)
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
if uploadErr != nil {
|
||||
t.Skipf("Could not create replicated volume: %v", uploadErr)
|
||||
return
|
||||
}
|
||||
t.Logf("Created replicated volume %d with collection %s, replication %s", volumeId, testCollection, testReplication)
|
||||
|
||||
// Acquire lock
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout")
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Execute EC encoding with the same collection
|
||||
ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
|
||||
args := []string{
|
||||
"-volumeId", fmt.Sprintf("%d", volumeId),
|
||||
"-collection", testCollection,
|
||||
"-force",
|
||||
}
|
||||
|
||||
outputStr, encodeErr := captureCommandOutput(t, ecEncodeCmd, args, commandEnv)
|
||||
t.Logf("EC encode output:\n%s", outputStr)
|
||||
|
||||
// For consistent replicas, should see "all X replicas are consistent"
|
||||
assert.Contains(t, outputStr, "replicas are consistent", "Should detect replicas are consistent")
|
||||
|
||||
if encodeErr != nil {
|
||||
t.Logf("EC encoding result: %v", encodeErr)
|
||||
} else {
|
||||
t.Log("EC encoding completed successfully")
|
||||
}
|
||||
})
|
||||
|
||||
// Test: Create divergent replicas and verify sync/union code is exercised
|
||||
t.Run("sync_replicated_volume_divergent", func(t *testing.T) {
|
||||
const testCollection = "replicated_divergent"
|
||||
const testReplication = "010" // 2 copies on different servers on the same rack
|
||||
|
||||
// Create replicated volume with initial data
|
||||
var volumeId needle.VolumeId
|
||||
var uploadErr error
|
||||
for retry := 0; retry < 5; retry++ {
|
||||
volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication)
|
||||
if uploadErr == nil {
|
||||
break
|
||||
}
|
||||
t.Logf("Upload attempt %d failed: %v, retrying...", retry+1, uploadErr)
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
if uploadErr != nil {
|
||||
t.Skipf("Could not create replicated volume: %v", uploadErr)
|
||||
return
|
||||
}
|
||||
t.Logf("Created replicated volume %d with collection %s, replication %s", volumeId, testCollection, testReplication)
|
||||
|
||||
// Get volume locations to identify the two replicas
|
||||
locations, found := commandEnv.MasterClient.GetLocationsClone(uint32(volumeId))
|
||||
if !found || len(locations) < 2 {
|
||||
t.Skipf("Could not get 2 replica locations for volume %d (got %d)", volumeId, len(locations))
|
||||
return
|
||||
}
|
||||
t.Logf("Volume %d has replicas at: %v and %v", volumeId, locations[0].Url, locations[1].Url)
|
||||
|
||||
// Write an extra entry to ONLY one replica to create divergence
|
||||
// We'll use the WriteNeedleBlob gRPC call to inject data directly
|
||||
extraNeedleId := uint64(999999) // Use a high needle ID unlikely to conflict
|
||||
extraData := []byte("extra data written to only one replica to create divergence")
|
||||
|
||||
err := injectNeedleToOneReplica(grpc.WithInsecure(), volumeId, locations[0], extraNeedleId, extraData)
|
||||
if err != nil {
|
||||
t.Logf("Could not inject divergent needle (may not be supported): %v", err)
|
||||
// Fall back to testing consistent path
|
||||
} else {
|
||||
t.Logf("Injected extra needle %d to replica %s to create divergence", extraNeedleId, locations[0].Url)
|
||||
}
|
||||
|
||||
// Acquire lock
|
||||
locked, unlock := tryLockWithTimeout(t, commandEnv, 30*time.Second)
|
||||
if !locked {
|
||||
t.Skip("Could not acquire lock within timeout")
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
// Execute EC encoding - should detect divergence and build union
|
||||
ecEncodeCmd := shell.Commands[findCommandIndex("ec.encode")]
|
||||
args := []string{
|
||||
"-volumeId", fmt.Sprintf("%d", volumeId),
|
||||
"-collection", testCollection,
|
||||
"-force",
|
||||
}
|
||||
|
||||
outputStr, encodeErr := captureCommandOutput(t, ecEncodeCmd, args, commandEnv)
|
||||
t.Logf("EC encode output:\n%s", outputStr)
|
||||
|
||||
// Check if divergence was detected
|
||||
if strings.Contains(outputStr, "building union") {
|
||||
t.Log("SUCCESS: Divergent replicas detected - sync/union code was exercised")
|
||||
assert.Contains(t, outputStr, "selected", "Should show which replica was selected as best")
|
||||
// The "copied" message only appears if the best replica is missing entries from other replicas
|
||||
// In our case, we injected into what becomes the best replica (higher file count),
|
||||
// so copying may not be needed. The key is that "building union" was triggered.
|
||||
if strings.Contains(outputStr, "copied") {
|
||||
t.Log("Entries were copied between replicas")
|
||||
} else {
|
||||
t.Log("Best replica already had all entries (injected entry was on best replica)")
|
||||
}
|
||||
} else if strings.Contains(outputStr, "replicas are consistent") {
|
||||
t.Log("Replicas were consistent (injection may not have worked)")
|
||||
} else {
|
||||
t.Log("Single replica or sync not triggered")
|
||||
}
|
||||
|
||||
if encodeErr != nil {
|
||||
t.Logf("EC encoding result: %v", encodeErr)
|
||||
} else {
|
||||
t.Log("EC encoding completed successfully")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// injectNeedleToOneReplica writes a needle directly to one replica to create divergence
|
||||
func injectNeedleToOneReplica(grpcDialOption grpc.DialOption, vid needle.VolumeId, location wdclient.Location, needleId uint64, data []byte) error {
|
||||
return operation.WithVolumeServerClient(false, location.ServerAddress(), grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
||||
blob, size := generateNeedleBlobV3(needleId, data)
|
||||
_, err := client.WriteNeedleBlob(context.Background(), &volume_server_pb.WriteNeedleBlobRequest{
|
||||
VolumeId: uint32(vid),
|
||||
NeedleId: needleId,
|
||||
Size: size,
|
||||
NeedleBlob: blob,
|
||||
})
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func generateNeedleBlobV3(needleId uint64, data []byte) ([]byte, int32) {
|
||||
// Overhead: DataSize(4) + Data(N) + Flags(1)
|
||||
size := 4 + len(data) + 1
|
||||
|
||||
// Create struct to calculate padding
|
||||
// Padding is based on the Size field (DataSize + Overhead)
|
||||
padding := needle.PaddingLength(types.Size(size), needle.Version3)
|
||||
|
||||
blob := make([]byte, 16+size+4+8+int(padding)) // Header(16) + Body(Size) + Checksum(4) + Timestamp(8) + Padding
|
||||
|
||||
// Header
|
||||
util.Uint32toBytes(blob[0:4], 0x12345678) // Cookie
|
||||
util.Uint64toBytes(blob[4:12], needleId)
|
||||
util.Uint32toBytes(blob[12:16], uint32(size))
|
||||
|
||||
// Body
|
||||
util.Uint32toBytes(blob[16:20], uint32(len(data)))
|
||||
copy(blob[20:], data)
|
||||
blob[20+len(data)] = 0 // Flags
|
||||
|
||||
// Checksum
|
||||
crc := needle.NewCRC(data)
|
||||
util.Uint32toBytes(blob[20+len(data)+1:20+len(data)+1+4], crc.Value())
|
||||
|
||||
// Timestamp (space reserved, filled by server)
|
||||
// Padding (zeroes)
|
||||
|
||||
return blob, int32(size)
|
||||
}
|
||||
|
||||
// uploadTestDataWithReplication uploads test data and returns the volume ID
|
||||
// using the specified collection and replication level. All files are uploaded to the same volume.
|
||||
func uploadTestDataWithReplication(t *testing.T, masterAddr string, collection string, replication string) (needle.VolumeId, error) {
|
||||
const numFiles = 20 // Reduced count since we're uploading to same volume
|
||||
|
||||
// Assign multiple file IDs from the same volume
|
||||
assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress {
|
||||
return pb.ServerAddress(masterAddr)
|
||||
}, grpc.WithInsecure(), &operation.VolumeAssignRequest{
|
||||
Count: uint64(numFiles),
|
||||
Collection: collection,
|
||||
Replication: replication,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to assign volume: %v", err)
|
||||
}
|
||||
|
||||
// Parse volume ID from file ID
|
||||
fid, err := needle.ParseFileIdFromString(assignResult.Fid)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to parse file ID: %v", err)
|
||||
}
|
||||
volumeId := fid.VolumeId
|
||||
|
||||
// Create uploader for upload
|
||||
uploader, err := operation.NewUploader()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to create uploader: %v", err)
|
||||
}
|
||||
|
||||
// Upload test data to the same volume using the assigned file IDs
|
||||
// The first FID is assignResult.Fid, subsequent ones increment the needle key
|
||||
baseNeedleKey := uint64(fid.Key)
|
||||
for i := 0; i < numFiles; i++ {
|
||||
data := []byte(fmt.Sprintf("test data for replicated volume sync test - entry %d - padding to make it larger %s",
|
||||
i, strings.Repeat("x", 1000)))
|
||||
|
||||
// Construct FID for this file (same volume, incrementing needle key)
|
||||
currentFid := fmt.Sprintf("%d,%x%08x", volumeId, baseNeedleKey+uint64(i), fid.Cookie)
|
||||
uploadUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, currentFid)
|
||||
|
||||
_, _, uploadErr := uploader.Upload(context.Background(), bytes.NewReader(data), &operation.UploadOption{
|
||||
UploadUrl: uploadUrl,
|
||||
Filename: fmt.Sprintf("file%d.txt", i),
|
||||
})
|
||||
if uploadErr != nil {
|
||||
t.Logf("Upload %d failed: %v", i, uploadErr)
|
||||
}
|
||||
}
|
||||
|
||||
return volumeId, nil
|
||||
}
|
||||
|
||||
@@ -5,14 +5,13 @@ ARG FOUNDATIONDB_VERSION=7.4.5
|
||||
ENV FOUNDATIONDB_VERSION=${FOUNDATIONDB_VERSION}
|
||||
|
||||
# Install build dependencies and download prebuilt FoundationDB clients
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN set -euo pipefail && \
|
||||
apt-get update && apt-get install -y \
|
||||
RUN apt-get update && apt-get install -y \
|
||||
build-essential \
|
||||
git \
|
||||
wget \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/* && \
|
||||
set -euo pipefail && \
|
||||
case "${FOUNDATIONDB_VERSION}" in \
|
||||
"7.4.5") EXPECTED_SHA256="f2176b86b7e1b561c3632b4e6e7efb82e3b8f57c2ff0d0ac4671e742867508aa" ;; \
|
||||
*) echo "ERROR: No known ARM64 client checksum for FoundationDB ${FOUNDATIONDB_VERSION}. Please update this Dockerfile." >&2; exit 1 ;; \
|
||||
|
||||
@@ -15,7 +15,6 @@ RUN apt-get update && apt-get install -y \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install FoundationDB server + client debs with checksum verification
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
RUN set -euo pipefail && \
|
||||
apt-get update && \
|
||||
case "${FOUNDATIONDB_VERSION}" in \
|
||||
|
||||
@@ -50,24 +50,6 @@ test-benchmark: ## Run performance benchmarks
|
||||
@echo "$(YELLOW)Running FoundationDB performance benchmarks...$(NC)"
|
||||
@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb -bench=. ./test/foundationdb/...
|
||||
|
||||
test-benchmark-filer: ## Run filer store benchmarks (batch vs no-batch comparison)
|
||||
@echo "$(YELLOW)Running FoundationDB filer store benchmarks...$(NC)"
|
||||
@echo "$(BLUE)Comparing batched vs non-batched write performance$(NC)"
|
||||
@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb \
|
||||
-bench='BenchmarkFoundationDBStore_.*' \
|
||||
-benchmem \
|
||||
-benchtime=5s \
|
||||
./weed/filer/foundationdb/...
|
||||
|
||||
test-benchmark-concurrent: ## Run concurrent operation benchmarks
|
||||
@echo "$(YELLOW)Running concurrent operation benchmarks...$(NC)"
|
||||
@cd ../../ && go test -v -timeout=$(TEST_TIMEOUT) -tags foundationdb \
|
||||
-bench='BenchmarkFoundationDBStore_Concurrent.*' \
|
||||
-benchmem \
|
||||
-benchtime=10s \
|
||||
-cpu=1,2,4,8 \
|
||||
./weed/filer/foundationdb/...
|
||||
|
||||
# ARM64 specific targets (Apple Silicon / M1/M2/M3 Macs)
|
||||
setup-arm64: ## Set up ARM64-native FoundationDB cluster (builds from source)
|
||||
@echo "$(YELLOW)Setting up ARM64-native FoundationDB cluster...$(NC)"
|
||||
|
||||
@@ -353,31 +353,12 @@ The tests are designed to be reliable in CI environments with:
|
||||
Run performance benchmarks:
|
||||
|
||||
```bash
|
||||
make test-benchmark # Run all benchmarks
|
||||
make test-benchmark-filer # Filer store benchmarks with batch comparison
|
||||
make test-benchmark-concurrent # Concurrent operation benchmarks (varies CPU count)
|
||||
make test-benchmark
|
||||
|
||||
# Sample expected results:
|
||||
# BenchmarkFoundationDBStore_InsertEntry-8 1000 1.2ms per op
|
||||
# BenchmarkFoundationDBStore_FindEntry-8 5000 0.5ms per op
|
||||
# BenchmarkFoundationDBStore_KvOperations-8 2000 0.8ms per op
|
||||
# BenchmarkFoundationDBStore_InsertEntry_NoBatch-8 1000 1.0ms per op (optimal for S3)
|
||||
# BenchmarkFoundationDBStore_InsertEntry_WithBatch-8 1200 0.9ms per op (bulk ingestion)
|
||||
# BenchmarkFoundationDBStore_ConcurrentInsert_*-8 5000 0.3ms per op (parallel writes)
|
||||
```
|
||||
|
||||
### Batch vs Non-Batch Performance
|
||||
|
||||
The FoundationDB filer store supports two write modes:
|
||||
|
||||
| Mode | Config | Best For | Latency | Throughput |
|
||||
|------|--------|----------|---------|------------|
|
||||
| **Direct Commit** (default) | `batch_enabled = false` | S3 API, low-latency workloads | ~1-5ms per op | Good |
|
||||
| **Batched** | `batch_enabled = true` | Bulk ingestion, high throughput | Variable | Higher |
|
||||
|
||||
Run the comparison benchmark:
|
||||
```bash
|
||||
make test-benchmark-filer
|
||||
# BenchmarkFoundationDBStore_InsertEntry-8 1000 1.2ms per op
|
||||
# BenchmarkFoundationDBStore_FindEntry-8 5000 0.5ms per op
|
||||
# BenchmarkFoundationDBStore_KvOperations-8 2000 0.8ms per op
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
@@ -28,8 +28,8 @@ services:
|
||||
echo 'testing:testing@fdb1:4500,fdb2:4502,fdb3:4504' > /var/fdb/config/fdb.cluster
|
||||
fi
|
||||
# Start FDB processes
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4501 --listen_address=0.0.0.0:4501 --coordination=fdb1:4500 &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb1:4500 --listen_address=0.0.0.0:4500 --coordination=fdb1:4500 --class=coordination &
|
||||
wait
|
||||
"
|
||||
|
||||
@@ -59,8 +59,8 @@ services:
|
||||
# Wait for cluster file from fdb1
|
||||
while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
|
||||
# Start FDB processes
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4503 --listen_address=0.0.0.0:4503 --coordination=fdb1:4500 &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb2:4502 --listen_address=0.0.0.0:4502 --coordination=fdb1:4500 --class=coordination &
|
||||
wait
|
||||
"
|
||||
|
||||
@@ -90,8 +90,8 @@ services:
|
||||
# Wait for cluster file from fdb1
|
||||
while [ ! -f /var/fdb/config/fdb.cluster ]; do sleep 1; done
|
||||
# Start FDB processes
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
|
||||
/usr/sbin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4505 --listen_address=0.0.0.0:4505 --coordination=fdb1:4500 &
|
||||
/usr/bin/fdbserver --config_path=/var/fdb/config --datadir=/var/fdb/data --logdir=/var/fdb/logs --public_address=fdb3:4504 --listen_address=0.0.0.0:4504 --coordination=fdb1:4500 --class=coordination &
|
||||
wait
|
||||
"
|
||||
|
||||
|
||||
@@ -1,8 +1,19 @@
|
||||
# FoundationDB Filer Configuration for Testing
|
||||
# FoundationDB Filer Configuration
|
||||
|
||||
[foundationdb]
|
||||
enabled = true
|
||||
cluster_file = "/var/fdb/config/fdb.cluster"
|
||||
# api_version = 740
|
||||
# timeout = "5s"
|
||||
# directory_prefix = "seaweedfs"
|
||||
api_version = 740
|
||||
timeout = "5s"
|
||||
max_retry_delay = "1s"
|
||||
directory_prefix = "seaweedfs"
|
||||
|
||||
# For testing different configurations
|
||||
[foundationdb.test]
|
||||
enabled = false
|
||||
cluster_file = "/var/fdb/config/fdb.cluster"
|
||||
api_version = 740
|
||||
timeout = "10s"
|
||||
max_retry_delay = "2s"
|
||||
directory_prefix = "seaweedfs_test"
|
||||
location = "/test"
|
||||
|
||||
@@ -12,20 +12,20 @@ COVERAGE_FILE := coverage.out
|
||||
# Check if weed binary exists
|
||||
check-binary:
|
||||
@if [ ! -f "$(WEED_BINARY)" ]; then \
|
||||
echo "ERROR SeaweedFS binary not found at $(WEED_BINARY)"; \
|
||||
echo "❌ SeaweedFS binary not found at $(WEED_BINARY)"; \
|
||||
echo " Please run 'make' in the root directory first"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "OK SeaweedFS binary found"
|
||||
@echo "✅ SeaweedFS binary found"
|
||||
|
||||
# Check FUSE installation
|
||||
check-fuse:
|
||||
@if command -v fusermount >/dev/null 2>&1; then \
|
||||
echo "OK FUSE is installed (Linux)"; \
|
||||
echo "✅ FUSE is installed (Linux)"; \
|
||||
elif command -v umount >/dev/null 2>&1 && [ "$$(uname)" = "Darwin" ]; then \
|
||||
echo "OK FUSE is available (macOS)"; \
|
||||
echo "✅ FUSE is available (macOS)"; \
|
||||
else \
|
||||
echo "ERROR FUSE not found. Please install:"; \
|
||||
echo "❌ FUSE not found. Please install:"; \
|
||||
echo " Ubuntu/Debian: sudo apt-get install fuse"; \
|
||||
echo " CentOS/RHEL: sudo yum install fuse"; \
|
||||
echo " macOS: brew install macfuse"; \
|
||||
@@ -36,8 +36,8 @@ check-fuse:
|
||||
check-go:
|
||||
@go version | grep -q "go1\.[2-9][0-9]" || \
|
||||
go version | grep -q "go1\.2[1-9]" || \
|
||||
(echo "ERROR Go $(GO_VERSION)+ required. Current: $$(go version)" && exit 1)
|
||||
@echo "OK Go version check passed"
|
||||
(echo "❌ Go $(GO_VERSION)+ required. Current: $$(go version)" && exit 1)
|
||||
@echo "✅ Go version check passed"
|
||||
|
||||
# Verify all prerequisites
|
||||
check-prereqs: check-go check-fuse
|
||||
@@ -45,9 +45,9 @@ check-prereqs: check-go check-fuse
|
||||
|
||||
# Build the SeaweedFS binary (if needed)
|
||||
build:
|
||||
@echo "Building SeaweedFS..."
|
||||
@echo "🔨 Building SeaweedFS..."
|
||||
cd ../.. && make
|
||||
@echo "OK Build complete"
|
||||
@echo "✅ Build complete"
|
||||
|
||||
# Initialize go module (if needed)
|
||||
init-module:
|
||||
|
||||
@@ -216,7 +216,7 @@ jobs:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: '1.24'
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Install FUSE
|
||||
run: sudo apt-get install -y fuse
|
||||
|
||||
@@ -3,7 +3,6 @@ package fuse_test
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
|
||||
@@ -3,5 +3,5 @@ package fuse_test
|
||||
import "testing"
|
||||
|
||||
func TestMinimal(t *testing.T) {
|
||||
t.Log("minimal test")
|
||||
t.Log("minimal test")
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
<junit.version>4.13.2</junit.version>
|
||||
<seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
|
||||
<jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
|
||||
<netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
|
||||
<netty.version>4.1.125.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
|
||||
<parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
|
||||
<parquet.format.version>2.12.0</parquet.format.version>
|
||||
<surefire.jvm.args>
|
||||
|
||||
@@ -172,8 +172,7 @@ services:
|
||||
- -master=seaweedfs-master:9333
|
||||
- -ip=seaweedfs-mq-broker
|
||||
- -port=17777
|
||||
- -debug
|
||||
- -debug.port=18777
|
||||
- -port.pprof=18777
|
||||
depends_on:
|
||||
seaweedfs-filer:
|
||||
condition: service_healthy
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
module github.com/seaweedfs/seaweedfs/test/kafka
|
||||
|
||||
go 1.24.9
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
|
||||
require (
|
||||
github.com/IBM/sarama v1.46.0
|
||||
@@ -14,7 +16,7 @@ require (
|
||||
replace github.com/seaweedfs/seaweedfs => ../../
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.17.0 // indirect
|
||||
cloud.google.com/go/auth v0.16.5 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
|
||||
@@ -41,26 +43,25 @@ require (
|
||||
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||
github.com/aws/aws-sdk-go v1.55.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 // indirect
|
||||
github.com/aws/smithy-go v1.23.2 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bradenaw/juniper v0.15.3 // indirect
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect
|
||||
@@ -81,7 +82,6 @@ require (
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/eapache/go-resiliency v1.7.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
|
||||
github.com/eapache/queue v1.1.0 // indirect
|
||||
@@ -115,7 +115,7 @@ require (
|
||||
github.com/google/btree v1.1.3 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
|
||||
github.com/gorilla/mux v1.8.1 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
@@ -161,7 +161,7 @@ require (
|
||||
github.com/oracle/oci-go-sdk/v65 v65.98.0 // indirect
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect
|
||||
github.com/panjf2000/ants/v2 v2.11.3 // indirect
|
||||
github.com/parquet-go/parquet-go v0.26.3 // indirect
|
||||
github.com/parquet-go/parquet-go v0.25.1 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
@@ -229,20 +229,20 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.38.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.46.0 // indirect
|
||||
golang.org/x/crypto v0.45.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect
|
||||
golang.org/x/image v0.34.0 // indirect
|
||||
golang.org/x/net v0.48.0 // indirect
|
||||
golang.org/x/oauth2 v0.34.0 // indirect
|
||||
golang.org/x/image v0.33.0 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/oauth2 v0.32.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/term v0.38.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/time v0.14.0 // indirect
|
||||
google.golang.org/api v0.258.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/term v0.37.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
google.golang.org/api v0.247.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
|
||||
google.golang.org/grpc/security/advancedtls v1.0.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
google.golang.org/protobuf v1.36.10 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/validator.v2 v2.0.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
||||
@@ -15,7 +15,6 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY
|
||||
cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
|
||||
cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI=
|
||||
cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ=
|
||||
cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
|
||||
@@ -103,47 +102,44 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
|
||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
|
||||
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
|
||||
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0 h1:/WMUA0kjhZExjOQN2z3oLALDREea1A7TobfuiBrKlwc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.0/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3/go.mod h1:br7KA6edAAqDGUYJ+zVVPAyMrPhnN+zdt17yTUT6FPw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs=
|
||||
github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
|
||||
github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bradenaw/juniper v0.15.3 h1:RHIAMEDTpvmzV1wg1jMAHGOoI2oJUSPx3lxRldXnFGo=
|
||||
@@ -206,8 +202,6 @@ github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXI
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M=
|
||||
github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
|
||||
github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA=
|
||||
github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws=
|
||||
@@ -357,7 +351,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
|
||||
@@ -508,7 +501,6 @@ github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZ
|
||||
github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
|
||||
github.com/parquet-go/parquet-go v0.25.1 h1:l7jJwNM0xrk0cnIIptWMtnSnuxRkwq53S+Po3KG8Xgo=
|
||||
github.com/parquet-go/parquet-go v0.25.1/go.mod h1:AXBuotO1XiBtcqJb/FKFyjBG4aqa3aQAAWF3ZPzCanY=
|
||||
github.com/parquet-go/parquet-go v0.26.3/go.mod h1:h9GcSt41Knf5qXI1tp1TfR8bDBUtvdUMzSKe26aZcHk=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
@@ -544,10 +536,10 @@ github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4
|
||||
github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 h1:Y258uzXU/potCYnQd1r6wlAnoMB68BiCkCcCnKx1SH8=
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8/go.mod h1:bSJjRokAHHOhA+XFxplld8w2R/dXLH7Z3BZ532vhFwU=
|
||||
github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
|
||||
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
|
||||
github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
|
||||
github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
|
||||
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
|
||||
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
|
||||
github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg=
|
||||
github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
|
||||
github.com/rclone/rclone v1.71.2 h1:3Jk5xNPFrZhVABRuN/OPvApuZQddpE2tkhYMuEn1Ud4=
|
||||
github.com/rclone/rclone v1.71.2/go.mod h1:dCK9FzPDlpkbQJ9M7MmWsmv3X5nibfWe+ogJXu6gSgM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
|
||||
@@ -716,6 +708,8 @@ go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJr
|
||||
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
|
||||
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
@@ -738,8 +732,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
|
||||
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
|
||||
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
|
||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -754,8 +748,8 @@ golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 h1:3yiSh9fhy5/RhCSntf4Sy0Tnx
|
||||
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50/go.mod h1:rT6SFzZ7oxADUDx58pcaKFTcZ+inxAa9fTrYx/uVYwg=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.34.0 h1:33gCkyw9hmwbZJeZkct8XyR11yH889EQt/QH4VmXMn8=
|
||||
golang.org/x/image v0.34.0/go.mod h1:2RNFBZRB+vnwwFil8GkMdRvrJOFd1AzdZI6vOY+eJVU=
|
||||
golang.org/x/image v0.33.0 h1:LXRZRnv1+zGd5XBUVRFmYEphyyKJjQjCRiOuAP3sZfQ=
|
||||
golang.org/x/image v0.33.0/go.mod h1:DD3OsTYT9chzuzTQt+zMcOlBHgfoKQb1gry8p76Y1sc=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
@@ -780,6 +774,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
|
||||
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -819,8 +815,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
|
||||
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -829,7 +825,6 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr
|
||||
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
|
||||
golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
|
||||
golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -899,8 +894,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
|
||||
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -913,8 +908,8 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
|
||||
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
|
||||
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
|
||||
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
|
||||
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -930,14 +925,13 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
|
||||
golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
|
||||
golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
|
||||
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
@@ -985,8 +979,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
|
||||
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
|
||||
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
|
||||
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@@ -1011,7 +1005,6 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr
|
||||
google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
|
||||
google.golang.org/api v0.247.0 h1:tSd/e0QrUlLsrwMKmkbQhYVa109qIintOls2Wh6bngc=
|
||||
google.golang.org/api v0.247.0/go.mod h1:r1qZOPmxXffXg6xS5uhx16Fa/UFY8QU/K4bfKrnvovM=
|
||||
google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
@@ -1053,7 +1046,6 @@ google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
@@ -1084,7 +1076,6 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
|
||||
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
|
||||
@@ -152,7 +152,7 @@ func testTwoConsumersRebalance(t *testing.T, addr, topicName, groupID string) {
|
||||
|
||||
// Wait for rebalancing to occur - both consumers should get new assignments
|
||||
var rebalancedAssignment1, rebalancedAssignment2 []int32
|
||||
|
||||
|
||||
// Consumer1 should get a rebalance assignment
|
||||
select {
|
||||
case partitions := <-handler1.assignments:
|
||||
@@ -372,7 +372,7 @@ func testMultipleConsumersJoin(t *testing.T, addr, topicName, groupID string) {
|
||||
t.Errorf("Partition %d assigned to multiple consumers", partition)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Each consumer should get exactly 1 partition (4 partitions / 4 consumers)
|
||||
if len(assignment) != 1 {
|
||||
t.Errorf("Consumer%d should get exactly 1 partition, got %d", i, len(assignment))
|
||||
@@ -408,7 +408,7 @@ func (h *RebalanceTestHandler) Setup(session sarama.ConsumerGroupSession) error
|
||||
h.readyOnce.Do(func() {
|
||||
close(h.ready)
|
||||
})
|
||||
|
||||
|
||||
// Send partition assignment
|
||||
partitions := make([]int32, 0)
|
||||
for topic, partitionList := range session.Claims() {
|
||||
@@ -417,13 +417,13 @@ func (h *RebalanceTestHandler) Setup(session sarama.ConsumerGroupSession) error
|
||||
partitions = append(partitions, partition)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
select {
|
||||
case h.assignments <- partitions:
|
||||
default:
|
||||
// Channel might be full, that's ok
|
||||
}
|
||||
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ func TestSchemaEndToEnd_AvroRoundTrip(t *testing.T) {
|
||||
// Verify all fields
|
||||
assert.Equal(t, int32(12345), decodedMap["id"])
|
||||
assert.Equal(t, "Alice Johnson", decodedMap["name"])
|
||||
|
||||
|
||||
// Verify union fields
|
||||
emailUnion, ok := decodedMap["email"].(map[string]interface{})
|
||||
require.True(t, ok, "Email should be a union")
|
||||
@@ -126,7 +126,7 @@ func TestSchemaEndToEnd_ProtobufRoundTrip(t *testing.T) {
|
||||
require.Equal(t, uint32(2), envelope.SchemaID, "Schema ID should match")
|
||||
// Note: ParseConfluentEnvelope defaults to FormatAvro; format detection requires schema registry
|
||||
require.Equal(t, schema.FormatAvro, envelope.Format, "Format defaults to Avro without schema registry lookup")
|
||||
|
||||
|
||||
// For Protobuf with indexes, we need to use the specialized parser
|
||||
protobufEnvelope, ok := schema.ParseConfluentProtobufEnvelopeWithIndexCount(confluentMsg, 1)
|
||||
require.True(t, ok, "Message should be a valid Protobuf envelope")
|
||||
@@ -269,6 +269,7 @@ func createMockSchemaRegistryForE2E(t *testing.T) *httptest.Server {
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
func getUserAvroSchemaForE2E() string {
|
||||
return `{
|
||||
"type": "record",
|
||||
|
||||
@@ -181,8 +181,7 @@ services:
|
||||
- -ip=seaweedfs-mq-broker
|
||||
- -port=17777
|
||||
- -logFlushInterval=0
|
||||
- -debug
|
||||
- -debug.port=18777
|
||||
- -port.pprof=18777
|
||||
depends_on:
|
||||
seaweedfs-filer:
|
||||
condition: service_healthy
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
)
|
||||
|
||||
// TestConsumerStallingPattern is a REPRODUCER for the consumer stalling bug.
|
||||
//
|
||||
//
|
||||
// This test simulates the exact pattern that causes consumers to stall:
|
||||
// 1. Consumer reads messages in batches
|
||||
// 2. Consumer commits offset after each batch
|
||||
@@ -24,7 +24,7 @@ import (
|
||||
// If the test PASSES, it means consumer successfully fetches all messages (bug fixed)
|
||||
func TestConsumerStallingPattern(t *testing.T) {
|
||||
t.Skip("REPRODUCER TEST: Requires running load test infrastructure. See comments for setup.")
|
||||
|
||||
|
||||
// This test documents the exact stalling pattern:
|
||||
// - Consumers consume messages 0-163, commit offset 163
|
||||
// - Next iteration: fetch offset 164+
|
||||
@@ -36,7 +36,7 @@ func TestConsumerStallingPattern(t *testing.T) {
|
||||
// 2. Empty fetch doesn't mean "end of partition" (could be transient)
|
||||
// 3. Consumer retries on empty fetch instead of giving up
|
||||
// 4. Logging shows why fetch stopped
|
||||
|
||||
|
||||
t.Logf("=== CONSUMER STALLING REPRODUCER ===")
|
||||
t.Logf("")
|
||||
t.Logf("Setup Steps:")
|
||||
@@ -72,27 +72,27 @@ func TestConsumerStallingPattern(t *testing.T) {
|
||||
// This is a UNIT reproducer that can run standalone
|
||||
func TestOffsetPlusOneCalculation(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
committedOffset int64
|
||||
name string
|
||||
committedOffset int64
|
||||
expectedNextOffset int64
|
||||
}{
|
||||
{"Offset 0", 0, 1},
|
||||
{"Offset 99", 99, 100},
|
||||
{"Offset 163", 163, 164}, // The exact stalling point!
|
||||
{"Offset 163", 163, 164}, // The exact stalling point!
|
||||
{"Offset 999", 999, 1000},
|
||||
{"Large offset", 10000, 10001},
|
||||
}
|
||||
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
// This is the critical calculation
|
||||
nextOffset := tc.committedOffset + 1
|
||||
|
||||
|
||||
if nextOffset != tc.expectedNextOffset {
|
||||
t.Fatalf("OFFSET MATH BUG: committed=%d, next=%d (expected %d)",
|
||||
tc.committedOffset, nextOffset, tc.expectedNextOffset)
|
||||
}
|
||||
|
||||
|
||||
t.Logf("✓ offset %d → next fetch at %d", tc.committedOffset, nextOffset)
|
||||
})
|
||||
}
|
||||
@@ -105,18 +105,18 @@ func TestEmptyFetchShouldNotStopConsumer(t *testing.T) {
|
||||
// Scenario: Consumer committed offset 163, then fetches 164+
|
||||
committedOffset := int64(163)
|
||||
nextFetchOffset := committedOffset + 1
|
||||
|
||||
|
||||
// First attempt: get empty (transient - data might not be available yet)
|
||||
// WRONG behavior (bug): Consumer sees 0 bytes and stops
|
||||
// wrongConsumerLogic := (firstFetchResult == 0) // gives up!
|
||||
|
||||
|
||||
// CORRECT behavior: Consumer should retry
|
||||
correctConsumerLogic := true // continues retrying
|
||||
|
||||
correctConsumerLogic := true // continues retrying
|
||||
|
||||
if !correctConsumerLogic {
|
||||
t.Fatalf("Consumer incorrectly gave up after empty fetch at offset %d", nextFetchOffset)
|
||||
}
|
||||
|
||||
|
||||
t.Logf("✓ Empty fetch doesn't stop consumer, continues retrying")
|
||||
})
|
||||
}
|
||||
|
||||