mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-08-29 04:07:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge master and resolve conflicts

Browse Source
This commit is contained in:
Chris Lu
2025-12-30 16:43:37 -08:00
parent 9318a1265e 0995214b37
commit 4081940720
195 changed files with 12242 additions and 2561 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/container_release_unified.yml
+18
View File
@@ -223,5 +223,23 @@ jobs:
echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub"
helm-release:
runs-on: ubuntu-latest
needs: [copy-to-dockerhub]
permissions:
contents: write
pages: write
steps:
- uses: actions/checkout@v6
- name: Publish Helm charts
uses: stefanprodan/helm-gh-pages@v1.7.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
charts_dir: k8s/charts
target_dir: helm
branch: gh-pages
helm_version: "3.18.4"
.github/workflows/go.yml
+9
View File
@@ -33,6 +33,15 @@ jobs:
run: |
cd weed; go get -v -t -d ./...
- name: Go Vet (excluding protobuf lock copying)
run: |
cd weed
# Run go vet and filter out known protobuf MessageState lock copying warnings
# These are expected in generated protobuf code with embedded sync.Mutex and are safe in practice
go vet -v ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-output.txt
# Fail only if there are actual vet errors (not counting the filtered lock warnings)
if grep -q "vet:" vet-output.txt; then exit 1; fi
- name: Build
run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v .
.github/workflows/helm_chart_release.yml
-23
View File
@@ -1,23 +0,0 @@
name: "helm: publish charts"
on:
push:
tags:
- '*'
permissions:
contents: write
pages: write
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
- name: Publish Helm charts
uses: stefanprodan/helm-gh-pages@v1.7.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
charts_dir: k8s/charts
target_dir: helm
branch: gh-pages
helm_version: "3.18.4"
.github/workflows/kafka-tests.yml
+7
View File
@@ -52,6 +52,7 @@ jobs:
run: |
cd test/kafka
go mod download
go mod tidy
- name: Run Kafka Gateway Unit Tests
run: |
@@ -96,6 +97,7 @@ jobs:
run: |
cd test/kafka
go mod download
go mod tidy
- name: Run Integration Tests
run: |
@@ -154,6 +156,7 @@ jobs:
cd test/kafka
# Use go mod download with timeout to prevent hanging
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
go mod tidy
- name: Build and start SeaweedFS MQ
run: |
@@ -332,6 +335,7 @@ jobs:
cd test/kafka
# Use go mod download with timeout to prevent hanging
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
go mod tidy
- name: Build and start SeaweedFS MQ
run: |
@@ -492,6 +496,7 @@ jobs:
run: |
cd test/kafka
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
go mod tidy
- name: Build and start SeaweedFS MQ
run: |
@@ -649,6 +654,7 @@ jobs:
run: |
cd test/kafka
timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules"
go mod tidy
- name: Build and start SeaweedFS MQ
run: |
@@ -803,6 +809,7 @@ jobs:
run: |
cd test/kafka
go mod download
go mod tidy
- name: Run Protocol Tests
run: |
.github/workflows/s3-sse-tests.yml
+50
View File
@@ -345,3 +345,53 @@ jobs:
name: s3-sse-performance-logs
path: test/s3/sse/weed-test*.log
retention-days: 7
s3-volume-encryption:
name: S3 Volume Encryption Test
runs-on: ubuntu-22.04
timeout-minutes: 20
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Install SeaweedFS
run: |
go install -buildvcs=false
- name: Run S3 Volume Encryption Integration Tests
timeout-minutes: 15
working-directory: test/s3/sse
run: |
set -x
echo "=== System Information ==="
uname -a
free -h
# Run volume encryption tests with -s3.encryptVolumeData flag
echo "🚀 Running S3 volume encryption integration tests..."
make test-volume-encryption || {
echo "❌ Volume encryption tests failed, checking logs..."
if [ -f /tmp/seaweedfs-sse-mini.log ]; then
echo "=== Server logs ==="
tail -100 /tmp/seaweedfs-sse-mini.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test)" || true
exit 1
}
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v6
with:
name: s3-volume-encryption-logs
path: /tmp/seaweedfs-sse-*.log
retention-days: 3
.github/workflows/test-s3-over-https-using-awscli.yml
+1 -1
View File
@@ -18,7 +18,7 @@ defaults:
jobs:
awscli-tests:
runs-on: ubuntu-latest
timeout-minutes: 5
timeout-minutes: 10
steps:
- uses: actions/checkout@v6
.gitignore
+6
View File
@@ -131,3 +131,9 @@ coverage.out
test/s3/remote_cache/remote-server.pid
test/s3/remote_cache/primary-server.pid
/test/erasure_coding/filerldb2
/test/s3/cors/test-mini-data
/test/s3/filer_group/test-volume-data
# ID and PID files
*.id
*.pid
go.mod
+29 -28
View File
@@ -1,8 +1,6 @@
module github.com/seaweedfs/seaweedfs
go 1.24.0
toolchain go1.24.1
go 1.24.9
require (
cloud.google.com/go v0.121.6 // indirect
@@ -36,7 +34,7 @@ require (
github.com/golang/snappy v1.0.0
github.com/google/btree v1.1.3
github.com/google/uuid v1.6.0
github.com/google/wire v0.6.0 // indirect
github.com/google/wire v0.7.0 // indirect
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
github.com/gorilla/mux v1.8.1
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
@@ -93,27 +91,27 @@ require (
go.etcd.io/etcd/client/v3 v3.6.6
go.mongodb.org/mongo-driver v1.17.6
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.43.0
gocloud.dev/pubsub/natspubsub v0.43.0
gocloud.dev v0.44.0
gocloud.dev/pubsub/natspubsub v0.44.0
gocloud.dev/pubsub/rabbitpubsub v0.43.0
golang.org/x/crypto v0.45.0
golang.org/x/crypto v0.46.0
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50
golang.org/x/image v0.34.0
golang.org/x/net v0.47.0
golang.org/x/oauth2 v0.32.0 // indirect
golang.org/x/sys v0.38.0
golang.org/x/net v0.48.0
golang.org/x/oauth2 v0.34.0 // indirect
golang.org/x/sys v0.39.0
golang.org/x/text v0.32.0 // indirect
golang.org/x/tools v0.39.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/api v0.247.0
google.golang.org/api v0.258.0
google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect
google.golang.org/grpc v1.77.0
google.golang.org/protobuf v1.36.10
google.golang.org/protobuf v1.36.11
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/mathutil v1.7.1
modernc.org/memory v1.11.0 // indirect
modernc.org/sqlite v1.39.0
modernc.org/sqlite v1.42.2
modernc.org/strutil v1.2.1
)
@@ -127,8 +125,8 @@ require (
github.com/arangodb/go-driver v1.6.9
github.com/armon/go-metrics v0.4.1
github.com/aws/aws-sdk-go-v2 v1.41.0
github.com/aws/aws-sdk-go-v2/config v1.31.3
github.com/aws/aws-sdk-go-v2/credentials v1.19.5
github.com/aws/aws-sdk-go-v2/config v1.32.6
github.com/aws/aws-sdk-go-v2/credentials v1.19.6
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
github.com/cognusion/imaging v1.0.2
github.com/fluent/fluent-logger-golang v1.10.1
@@ -147,18 +145,18 @@ require (
github.com/mattn/go-sqlite3 v1.14.32
github.com/minio/crc64nvme v1.1.1
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/parquet-go/parquet-go v0.25.1
github.com/parquet-go/parquet-go v0.26.3
github.com/pkg/sftp v1.13.10
github.com/rabbitmq/amqp091-go v1.10.0
github.com/rclone/rclone v1.71.2
github.com/rdleal/intervalst v1.5.0
github.com/redis/go-redis/v9 v9.17.2
github.com/schollz/progressbar/v3 v3.18.0
github.com/schollz/progressbar/v3 v3.19.0
github.com/shirou/gopsutil/v4 v4.25.11
github.com/tarantool/go-tarantool/v2 v2.4.1
github.com/tikv/client-go/v2 v2.0.7
github.com/xeipuuv/gojsonschema v1.2.0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1
github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0
go.etcd.io/etcd/client/pkg/v3 v3.6.6
go.uber.org/atomic v1.11.0
@@ -171,8 +169,9 @@ require github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // ind
require (
cloud.google.com/go/longrunning v0.6.7 // indirect
cloud.google.com/go/pubsub/v2 v2.0.0 // indirect
cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
github.com/bazelbuild/rules_go v0.46.0 // indirect
github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect
github.com/blevesearch/snowballstem v0.9.0 // indirect
@@ -201,6 +200,8 @@ require (
github.com/kr/text v0.2.0 // indirect
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
github.com/openzipkin/zipkin-go v0.4.3 // indirect
github.com/parquet-go/bitpack v1.0.0 // indirect
github.com/parquet-go/jsonlite v1.0.0 // indirect
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
github.com/pierrre/geohash v1.0.0 // indirect
github.com/quic-go/qpack v0.6.0 // indirect
@@ -226,7 +227,7 @@ require (
require (
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go/auth v0.16.5 // indirect
cloud.google.com/go/auth v0.17.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.5.2 // indirect
@@ -261,10 +262,10 @@ require (
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
@@ -272,7 +273,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
@@ -327,7 +328,7 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
github.com/gorilla/context v1.1.2 // indirect
github.com/gorilla/schema v1.4.1 // indirect
github.com/gorilla/securecookie v1.1.2 // indirect
@@ -445,15 +446,15 @@ require (
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/arch v0.20.0 // indirect
golang.org/x/term v0.37.0 // indirect
golang.org/x/time v0.12.0 // indirect
golang.org/x/term v0.38.0 // indirect
golang.org/x/time v0.14.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/validator.v2 v2.0.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.66.3 // indirect
modernc.org/libc v1.66.10 // indirect
moul.io/http2curl/v2 v2.3.0 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
storj.io/common v0.0.0-20250808122759-804533d519c1 // indirect
go.sum
+62 -63
View File
@@ -86,8 +86,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo
cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo=
cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0=
cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E=
cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI=
cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ=
cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4=
cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ=
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0=
@@ -385,8 +385,8 @@ cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhz
cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM=
cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk=
cloud.google.com/go/pubsub/v2 v2.0.0 h1:0qS6mRJ41gD1lNmM/vdm6bR7DQu6coQcVwD+VPf0Bz0=
cloud.google.com/go/pubsub/v2 v2.0.0/go.mod h1:0aztFxNzVQIRSZ8vUr79uH2bS3jwLebwK6q1sgEub+E=
cloud.google.com/go/pubsub/v2 v2.2.1 h1:3brZcshL3fIiD1qOxAE2QW9wxsfjioy014x4yC9XuYI=
cloud.google.com/go/pubsub/v2 v2.2.1/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
@@ -669,20 +669,20 @@ github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgP
github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
github.com/aws/aws-sdk-go-v2/credentials v1.19.5 h1:xMo63RlqP3ZZydpJDMBsH9uJ10hgHYfQFIk1cHDXrR4=
github.com/aws/aws-sdk-go-v2/credentials v1.19.5/go.mod h1:hhbH6oRcou+LpXfA/0vPElh/e0M3aFeOblE1sssAAEk=
github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3 h1:4GNV1lhyELGjMz5ILMRxDvxvOaeo3Ux9Z69S1EgVMMQ=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3/go.mod h1:br7KA6edAAqDGUYJ+zVVPAyMrPhnN+zdt17yTUT6FPw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
@@ -695,12 +695,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1c
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 h1:eYnlt6QxnFINKzwxP5/Ucs1vkG7VT3Iezmvfgc2waUw=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.7/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
@@ -1132,22 +1134,21 @@ github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI=
github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA=
github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4=
github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18=
github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4=
github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ=
github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
@@ -1469,8 +1470,12 @@ github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q
github.com/ory/dockertest/v3 v3.6.0/go.mod h1:4ZOpj8qBUmh8fcBSVzkH2bws2s91JdGvHUqan4GHEuQ=
github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
github.com/parquet-go/parquet-go v0.25.1 h1:l7jJwNM0xrk0cnIIptWMtnSnuxRkwq53S+Po3KG8Xgo=
github.com/parquet-go/parquet-go v0.25.1/go.mod h1:AXBuotO1XiBtcqJb/FKFyjBG4aqa3aQAAWF3ZPzCanY=
github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA=
github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs=
github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU=
github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0=
github.com/parquet-go/parquet-go v0.26.3 h1:kJY+xmjcR7BH77tyHqasJpIl3kch/6EIO3TW4tFj69M=
github.com/parquet-go/parquet-go v0.26.3/go.mod h1:h9GcSt41Knf5qXI1tp1TfR8bDBUtvdUMzSKe26aZcHk=
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
@@ -1612,8 +1617,8 @@ github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI=
github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0=
github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM=
github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA=
github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc=
github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec=
github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138 h1:bX1vBF7GQjPeFQsCAZ8gCQGS/nJQnekL7gZ4Qg/pF4E=
github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0=
github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30=
@@ -1777,8 +1782,8 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mo
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 h1:/NyPd9KnCJgzrEXCArqk1ThqCH2Dh31uUwl88o/VkuM=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM=
github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM=
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0=
github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0 h1:uvqheUfoEJz0CLLLaYct69wpvoaosM/2joWFcbwoqfw=
@@ -1886,10 +1891,10 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
gocloud.dev v0.43.0 h1:aW3eq4RMyehbJ54PMsh4hsp7iX8cO/98ZRzJJOzN/5M=
gocloud.dev v0.43.0/go.mod h1:eD8rkg7LhKUHrzkEdLTZ+Ty/vgPHPCd+yMQdfelQVu4=
gocloud.dev/pubsub/natspubsub v0.43.0 h1:k35tFoaorvD9Fa26zVEEzyXiMOEyXNHc0pBOmRYvQI0=
gocloud.dev/pubsub/natspubsub v0.43.0/go.mod h1:xJn8TO8pGYieDn6AsRFsYfhQW8cnC+xGmG9APGNxkpQ=
gocloud.dev v0.44.0 h1:iVyMAqFl2r6xUy7M4mfqwlN+21UpJoEtgHEcfiLMUXs=
gocloud.dev v0.44.0/go.mod h1:ZmjROXGdC/eKZLF1N+RujDlFRx3D+4Av2thREKDMVxY=
gocloud.dev/pubsub/natspubsub v0.44.0 h1:1Us76ckkdgtiE1p1rJZ+38b9TQP051bmjAiQlFQzYrM=
gocloud.dev/pubsub/natspubsub v0.44.0/go.mod h1:PvVAGIhL14PWGwWIXX/zAK42ixr2/PKP4Q4yMiAUraQ=
gocloud.dev/pubsub/rabbitpubsub v0.43.0 h1:6nNZFSlJ1dk2GujL8PFltfLz3vC6IbrpjGS4FTduo1s=
gocloud.dev/pubsub/rabbitpubsub v0.43.0/go.mod h1:sEaueAGat+OASRoB3QDkghCtibKttgg7X6zsPTm1pl0=
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
@@ -1911,14 +1916,13 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1982,7 +1986,6 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=
@@ -2052,12 +2055,11 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2086,8 +2088,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY=
golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -2212,14 +2214,13 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2231,14 +2232,13 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2267,8 +2267,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -2339,7 +2339,6 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg=
golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ=
golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
@@ -2420,8 +2419,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/
google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI=
google.golang.org/api v0.247.0 h1:tSd/e0QrUlLsrwMKmkbQhYVa109qIintOls2Wh6bngc=
google.golang.org/api v0.247.0/go.mod h1:r1qZOPmxXffXg6xS5uhx16Fa/UFY8QU/K4bfKrnvovM=
google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc=
google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -2559,8 +2558,8 @@ google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 h1:Nt6z9UHqSlIdIGJ
google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79/go.mod h1:kTmlBHMPqR5uCZPBvwa2B18mvubkjyY3CRLI0c6fj0s=
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4=
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -2624,8 +2623,8 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -2676,19 +2675,19 @@ modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=
modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM=
modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4=
modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc=
modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw=
modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ=
modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws=
modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo=
modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU=
modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE=
modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A=
modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q=
modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM=
modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA=
modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc=
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
@@ -2701,8 +2700,8 @@ modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0=
modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s=
modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ=
modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8=
modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A=
modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I=
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
@@ -2721,8 +2720,8 @@ modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4=
modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY=
modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E=
modernc.org/sqlite v1.42.2 h1:7hkZUNJvJFN2PgfUdjni9Kbvd4ef4mNLOu0B9FGxM74=
modernc.org/sqlite v1.42.2/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8=
modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
k8s/charts/seaweedfs/Chart.yaml
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.03"
appVersion: "4.04"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.0.403
version: 4.0.404
k8s/charts/seaweedfs/templates/volume/volume-statefulset.yaml
+3
View File
@@ -176,6 +176,9 @@ spec:
{{- if $volume.dataCenter }}
-dataCenter={{ $volume.dataCenter }} \
{{- end }}
{{- if $volume.id }}
-id={{ $volume.id }} \
{{- end }}
-ip.bind={{ $volume.ipBind }} \
-readMode={{ $volume.readMode }} \
{{- if $volume.whiteList }}
k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml
+17 -6
View File
@@ -134,11 +134,14 @@ spec:
{{- if .Values.worker.adminServer }}
-admin={{ .Values.worker.adminServer }} \
{{- else }}
-admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.grpcPort }} \
-admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }} \
{{- end }}
-capabilities={{ .Values.worker.capabilities }} \
-maxConcurrent={{ .Values.worker.maxConcurrent }} \
-workingDir={{ .Values.worker.workingDir }}{{- if .Values.worker.extraArgs }} \{{ end }}
-workingDir={{ .Values.worker.workingDir }}{{- if or .Values.worker.metricsPort .Values.worker.extraArgs }} \{{ end }}
{{- if .Values.worker.metricsPort }}
-metricsPort={{ .Values.worker.metricsPort }}{{- if .Values.worker.extraArgs }} \{{ end }}
{{- end }}
{{- range $index, $arg := .Values.worker.extraArgs }}
{{ $arg }}{{- if lt $index (sub (len $.Values.worker.extraArgs) 1) }} \{{ end }}
{{- end }}
@@ -187,9 +190,13 @@ spec:
{{- end }}
{{- if .Values.worker.livenessProbe.enabled }}
livenessProbe:
{{- with .Values.worker.livenessProbe.tcpSocket }}
{{- if .Values.worker.livenessProbe.httpGet }}
httpGet:
path: {{ .Values.worker.livenessProbe.httpGet.path }}
port: {{ .Values.worker.livenessProbe.httpGet.port }}
{{- else if .Values.worker.livenessProbe.tcpSocket }}
tcpSocket:
port: {{ .port }}
port: {{ .Values.worker.livenessProbe.tcpSocket.port }}
{{- end }}
initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }}
@@ -199,9 +206,13 @@ spec:
{{- end }}
{{- if .Values.worker.readinessProbe.enabled }}
readinessProbe:
{{- with .Values.worker.readinessProbe.tcpSocket }}
{{- if .Values.worker.readinessProbe.httpGet }}
httpGet:
path: {{ .Values.worker.readinessProbe.httpGet.path }}
port: {{ .Values.worker.readinessProbe.httpGet.port }}
{{- else if .Values.worker.readinessProbe.tcpSocket }}
tcpSocket:
port: {{ .port }}
port: {{ .Values.worker.readinessProbe.tcpSocket.port }}
{{- end }}
initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
k8s/charts/seaweedfs/values.yaml
+9 -3
View File
@@ -401,6 +401,10 @@ volume:
# Volume server's rack name
rack: null
# Stable identifier for the volume server, independent of IP address
# Useful for Kubernetes environments with hostPath volumes to maintain stable identity
id: null
# Volume server's data center name
dataCenter: null
@@ -1302,10 +1306,11 @@ worker:
extraEnvironmentVars: {}
# Health checks for worker pods
# Since workers do not have an HTTP endpoint, a tcpSocket probe on the metrics port is recommended.
# Workers expose /health (liveness) and /ready (readiness) endpoints on the metricsPort
livenessProbe:
enabled: true
tcpSocket:
httpGet:
path: /health
port: metrics
initialDelaySeconds: 30
periodSeconds: 60
@@ -1315,7 +1320,8 @@ worker:
readinessProbe:
enabled: true
tcpSocket:
httpGet:
path: /ready
port: metrics
initialDelaySeconds: 20
periodSeconds: 15
other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
+1 -1
View File
@@ -87,7 +87,7 @@ public class FilerGrpcClient {
.maxHeaderListSize(16 * 1024 * 1024)
.keepAliveTime(KEEP_ALIVE_TIME_SECONDS, TimeUnit.SECONDS)
.keepAliveTimeout(KEEP_ALIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS)
.keepAliveWithoutCalls(true)
.keepAliveWithoutCalls(false)
.withOption(io.grpc.netty.shaded.io.netty.channel.ChannelOption.SO_RCVBUF, 16 * 1024 * 1024)
.withOption(io.grpc.netty.shaded.io.netty.channel.ChannelOption.SO_SNDBUF, 16 * 1024 * 1024);
test/fuse_integration/Makefile
+9 -9
View File
@@ -12,20 +12,20 @@ COVERAGE_FILE := coverage.out
# Check if weed binary exists
check-binary:
@if [ ! -f "$(WEED_BINARY)" ]; then \
echo "❌ SeaweedFS binary not found at $(WEED_BINARY)"; \
echo "ERROR SeaweedFS binary not found at $(WEED_BINARY)"; \
echo " Please run 'make' in the root directory first"; \
exit 1; \
fi
@echo "✅ SeaweedFS binary found"
@echo "OK SeaweedFS binary found"
# Check FUSE installation
check-fuse:
@if command -v fusermount >/dev/null 2>&1; then \
echo "✅ FUSE is installed (Linux)"; \
echo "OK FUSE is installed (Linux)"; \
elif command -v umount >/dev/null 2>&1 && [ "$$(uname)" = "Darwin" ]; then \
echo "✅ FUSE is available (macOS)"; \
echo "OK FUSE is available (macOS)"; \
else \
echo "❌ FUSE not found. Please install:"; \
echo "ERROR FUSE not found. Please install:"; \
echo " Ubuntu/Debian: sudo apt-get install fuse"; \
echo " CentOS/RHEL: sudo yum install fuse"; \
echo " macOS: brew install macfuse"; \
@@ -36,8 +36,8 @@ check-fuse:
check-go:
@go version | grep -q "go1\.[2-9][0-9]" || \
go version | grep -q "go1\.2[1-9]" || \
(echo "❌ Go $(GO_VERSION)+ required. Current: $$(go version)" && exit 1)
@echo "✅ Go version check passed"
(echo "ERROR Go $(GO_VERSION)+ required. Current: $$(go version)" && exit 1)
@echo "OK Go version check passed"
# Verify all prerequisites
check-prereqs: check-go check-fuse
@@ -45,9 +45,9 @@ check-prereqs: check-go check-fuse
# Build the SeaweedFS binary (if needed)
build:
@echo "🔨 Building SeaweedFS..."
@echo "Building SeaweedFS..."
cd ../.. && make
@echo "✅ Build complete"
@echo "OK Build complete"
# Initialize go module (if needed)
init-module:
test/kafka/go.mod
+17 -15
View File
@@ -43,24 +43,25 @@ require (
github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go v1.55.8 // indirect
github.com/aws/aws-sdk-go-v2 v1.40.1 // indirect
github.com/aws/aws-sdk-go-v2 v1.41.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.31.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.3 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.6 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.6 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
@@ -82,6 +83,7 @@ require (
github.com/cronokirby/saferith v0.33.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/eapache/go-resiliency v1.7.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
github.com/eapache/queue v1.1.0 // indirect
@@ -229,14 +231,14 @@ require (
go.opentelemetry.io/otel/trace v1.38.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.45.0 // indirect
golang.org/x/crypto v0.46.0 // indirect
golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect
golang.org/x/image v0.34.0 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/net v0.48.0 // indirect
golang.org/x/oauth2 v0.32.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/term v0.37.0 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/term v0.38.0 // indirect
golang.org/x/text v0.32.0 // indirect
golang.org/x/time v0.12.0 // indirect
google.golang.org/api v0.247.0 // indirect
test/kafka/go.sum
+34 -30
View File
@@ -102,42 +102,44 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc=
github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco=
github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE=
github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas=
github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo=
github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8=
github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI=
github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE=
github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk=
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -202,6 +204,8 @@ github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXI
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M=
github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA=
github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws=
@@ -730,8 +734,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -811,8 +815,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -890,8 +894,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -904,8 +908,8 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
test/kafka/loadtest/mock_million_record_test.go
+18 -10
View File
@@ -14,6 +14,8 @@ import (
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/keepalive"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
@@ -119,22 +121,28 @@ type PublisherSession struct {
}
func NewDirectBrokerClient(brokerAddr string) (*DirectBrokerClient, error) {
ctx, cancel := context.WithCancel(context.Background())
// Use a short-lived context for dialing so we don't store a canceled
// context in the returned client. The client's operational context
// (used by methods) should be cancellable independently.
dialCtx, dialCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer dialCancel()
// Add connection timeout and keepalive settings
conn, err := grpc.DialContext(ctx, brokerAddr,
// Add keepalive settings; use exported server constants to keep values in sync.
conn, err := grpc.DialContext(dialCtx, brokerAddr,
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithTimeout(30*time.Second),
grpc.WithKeepaliveParams(keepalive.ClientParameters{
Time: 30 * time.Second, // Increased from 10s to 30s
Timeout: 10 * time.Second, // Increased from 5s to 10s
PermitWithoutStream: false, // Changed to false to reduce pings
Time: pb.GrpcKeepAliveTime, // align with server MinTime
Timeout: pb.GrpcKeepAliveTimeout, // align with server timeout
PermitWithoutStream: false, // reduce pings when idle
}))
if err != nil {
cancel()
return nil, fmt.Errorf("failed to connect to broker: %v", err)
}
// Create a long-lived context for the client's lifetime and store it
// in the returned DirectBrokerClient so callers can cancel when done.
clientCtx, clientCancel := context.WithCancel(context.Background())
client := mq_pb.NewSeaweedMessagingClient(conn)
return &DirectBrokerClient{
@@ -142,8 +150,8 @@ func NewDirectBrokerClient(brokerAddr string) (*DirectBrokerClient, error) {
conn: conn,
client: client,
publishers: make(map[string]*PublisherSession),
ctx: ctx,
cancel: cancel,
ctx: clientCtx,
cancel: clientCancel,
}, nil
}
test/s3/acl/s3_acl_versioning_test.go
+335
View File
@@ -0,0 +1,335 @@
package acl
import (
"context"
"os"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func getS3Client(t *testing.T) *s3.Client {
endpoint := os.Getenv("S3_ENDPOINT")
if endpoint == "" {
endpoint = "http://localhost:8333"
}
accessKey := os.Getenv("S3_ACCESS_KEY")
if accessKey == "" {
accessKey = "some_access_key1"
}
secretKey := os.Getenv("S3_SECRET_KEY")
if secretKey == "" {
secretKey = "some_secret_key1"
}
cfg, err := config.LoadDefaultConfig(
context.TODO(),
config.WithRegion("us-east-1"),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
accessKey,
secretKey,
"",
)),
)
require.NoError(t, err)
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = true
o.BaseEndpoint = aws.String(endpoint)
})
}
func createVersionedTestBucket(t *testing.T, client *s3.Client) string {
bucketName := "test-acl-versioned-" + strings.ToLower(strings.ReplaceAll(time.Now().Format("2006-01-02-15-04-05.000"), ":", "-"))
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
Bucket: aws.String(bucketName),
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
require.NoError(t, err)
time.Sleep(100 * time.Millisecond)
return bucketName
}
func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) {
listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{
Bucket: aws.String(bucketName),
})
if err == nil {
for _, obj := range listResp.Contents {
client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: obj.Key,
})
}
}
listVersionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucketName),
})
if err == nil {
for _, version := range listVersionsResp.Versions {
client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: version.Key,
VersionId: version.VersionId,
})
}
for _, marker := range listVersionsResp.DeleteMarkers {
client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: marker.Key,
VersionId: marker.VersionId,
})
}
}
client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
Bucket: aws.String(bucketName),
})
}
// TestGetObjectAclOnVersionedBucket tests retrieving ACL from versioned objects
func TestGetObjectAclOnVersionedBucket(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
objectKey := "versioned-object-acl"
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: strings.NewReader("Hello, ACL World!"),
})
require.NoError(t, err)
aclResp, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err)
assert.NotNil(t, aclResp.Owner)
t.Logf("Successfully retrieved ACL for versioned object %s (versionId: %s)", objectKey, *putResp.VersionId)
}
// TestGetObjectAclOnSpecificVersionInVersionedBucket tests retrieving ACL for specific versions
func TestGetObjectAclOnSpecificVersionInVersionedBucket(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
objectKey := "multi-version-object-acl"
version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: strings.NewReader("Version 1"),
})
require.NoError(t, err)
time.Sleep(50 * time.Millisecond)
version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: strings.NewReader("Version 2"),
})
require.NoError(t, err)
versionId1 := *version1Resp.VersionId
versionId2 := *version2Resp.VersionId
aclResp1, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
})
require.NoError(t, err)
assert.NotNil(t, aclResp1.Owner)
aclResp2, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
})
require.NoError(t, err)
assert.NotNil(t, aclResp2.Owner)
t.Logf("Successfully retrieved ACL for both versions: v1=%s, v2=%s", versionId1, versionId2)
}
// TestPutObjectAclOnVersionedBucket tests setting ACL on versioned objects
func TestPutObjectAclOnVersionedBucket(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
objectKey := "versioned-object-put-acl"
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: strings.NewReader("Hello, Put ACL!"),
})
require.NoError(t, err)
_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
ACL: types.ObjectCannedACLPublicRead,
})
require.NoError(t, err)
aclResp, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err)
assert.NotNil(t, aclResp.Owner)
t.Logf("Successfully set and verified ACL for versioned object %s (versionId: %s)", objectKey, *putResp.VersionId)
}
// TestPutObjectAclOnSpecificVersionInVersionedBucket tests setting ACL on specific versions
func TestPutObjectAclOnSpecificVersionInVersionedBucket(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
objectKey := "multi-version-object-put-acl"
version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: strings.NewReader("Version 1"),
})
require.NoError(t, err)
time.Sleep(50 * time.Millisecond)
version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: strings.NewReader("Version 2"),
})
require.NoError(t, err)
versionId1 := *version1Resp.VersionId
versionId2 := *version2Resp.VersionId
_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
ACL: types.ObjectCannedACLPublicRead,
})
require.NoError(t, err)
_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
ACL: types.ObjectCannedACLPrivate,
})
require.NoError(t, err)
aclResp1, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
})
require.NoError(t, err)
assert.NotNil(t, aclResp1.Owner)
aclResp2, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
})
require.NoError(t, err)
assert.NotNil(t, aclResp2.Owner)
t.Logf("Successfully set ACL on both versions: v1=%s, v2=%s", versionId1, versionId2)
}
// TestModifyAclOnDifferentVersionsIndependently tests that ACL changes on one version don't affect others
func TestModifyAclOnDifferentVersionsIndependently(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
objectKey := "versioned-independent-acl"
version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: strings.NewReader("Version 1"),
})
require.NoError(t, err)
time.Sleep(50 * time.Millisecond)
version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: strings.NewReader("Version 2"),
})
require.NoError(t, err)
versionId1 := *version1Resp.VersionId
versionId2 := *version2Resp.VersionId
_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
ACL: types.ObjectCannedACLPublicRead,
})
require.NoError(t, err)
_, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
ACL: types.ObjectCannedACLPrivate,
})
require.NoError(t, err)
aclRespLatest, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err)
assert.NotNil(t, aclRespLatest.Owner)
aclResp1, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
})
require.NoError(t, err)
assert.NotNil(t, aclResp1.Owner)
aclResp2, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
})
require.NoError(t, err)
assert.NotNil(t, aclResp2.Owner)
t.Logf("Successfully verified independent ACL management across versions")
}
test/s3/compatibility/run.sh
+1 -4
View File
@@ -22,10 +22,7 @@ docker stop $CONTAINER_NAME || echo "already stopped"
ulimit -n 10000
# Start weed w/ filer + s3 in the background
$WEED_BIN server \
-filer \
-s3 \
-volume.max 0 \
$WEED_BIN mini \
-master.volumeSizeLimitMB 5 \
-dir "$(pwd)/tmp" \
1>&2>weed.log &
test/s3/copying/Makefile
+19 -29
View File
@@ -63,30 +63,24 @@ start-seaweedfs: check-binary
@pkill -f "weed volume" || true
@pkill -f "weed filer" || true
@pkill -f "weed s3" || true
@pkill -f "weed mini" || true
@sleep 2
# Create necessary directories
@mkdir -p /tmp/seaweedfs-test-copying-master
@mkdir -p /tmp/seaweedfs-test-copying-volume
@mkdir -p /tmp/seaweedfs-test-copying
# Start master server with volume size limit
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -mdir=/tmp/seaweedfs-test-copying-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-master.log 2>&1 &
@sleep 3
# Start volume server
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-copying-volume -ip=127.0.0.1 > /tmp/seaweedfs-volume.log 2>&1 &
@sleep 3
# Start filer server (using standard SeaweedFS gRPC port convention: HTTP port + 10000)
@nohup $(SEAWEEDFS_BINARY) filer -port=$(FILER_PORT) -port.grpc=$$(( $(FILER_PORT) + 10000 )) -master=127.0.0.1:$(MASTER_PORT) -ip=127.0.0.1 > /tmp/seaweedfs-filer.log 2>&1 &
@sleep 3
# Create S3 configuration
@echo '{"identities":[{"name":"$(ACCESS_KEY)","credentials":[{"accessKey":"$(ACCESS_KEY)","secretKey":"$(SECRET_KEY)"}],"actions":["Admin","Read","Write"]}]}' > /tmp/seaweedfs-s3.json
# Start S3 server
@nohup $(SEAWEEDFS_BINARY) s3 -port=$(S3_PORT) -filer=127.0.0.1:$(FILER_PORT) -config=/tmp/seaweedfs-s3.json -ip.bind=127.0.0.1 > /tmp/seaweedfs-s3.log 2>&1 &
@sleep 5
# Start weed mini
@echo "Starting weed mini with dir=/tmp/seaweedfs-test-copying"
@export AWS_ACCESS_KEY_ID=$(ACCESS_KEY) && \
export AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) && \
# Start weed mini with S3 configuration
@echo "Starting weed mini..."
@nohup $(SEAWEEDFS_BINARY) mini \
-dir=/tmp/seaweedfs-test-copying \
-s3.port=$(S3_PORT) \
-s3.config=/tmp/seaweedfs-s3.json \
-ip=127.0.0.1 \
> /tmp/seaweedfs-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid 5
# Wait for S3 service to be ready
@echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)"
@@ -98,16 +92,12 @@ start-seaweedfs: check-binary
echo "Waiting for S3 service... ($$i/30)"; \
sleep 1; \
done
# Additional wait for filer gRPC to be ready
@echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)"
@sleep 2
@echo "$(GREEN)SeaweedFS server started successfully$(NC)"
@echo "Master: http://localhost:$(MASTER_PORT)"
@echo "Volume: http://localhost:$(VOLUME_PORT)"
@echo "Filer: http://localhost:$(FILER_PORT)"
# Additional wait for filer gRPC to be ready
@echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)"
@sleep 2
@echo "$(GREEN)SeaweedFS server started successfully$(NC)"
@echo "Mini Log: /tmp/seaweedfs-mini.log"
@echo "S3: http://localhost:$(S3_PORT)"
@echo "Volume Max Size: $(VOLUME_MAX_SIZE_MB)MB"
stop-seaweedfs:
@echo "$(YELLOW)Stopping SeaweedFS server...$(NC)"
test/s3/cors/Makefile
+25 -21
View File
@@ -11,6 +11,7 @@ VOLUME_PORT := 8080
FILER_PORT := 8888
TEST_TIMEOUT := 10m
TEST_PATTERN := TestCORS
SERVER_DIR := test-mini-data
# Default target
help:
@@ -41,21 +42,21 @@ build-weed:
@echo "Building SeaweedFS binary..."
@cd ../../../weed && go build -o weed_binary .
@chmod +x $(WEED_BINARY)
@echo "✅ SeaweedFS binary built at $(WEED_BINARY)"
@echo "OK SeaweedFS binary built at $(WEED_BINARY)"
check-deps: build-weed
@echo "Checking dependencies..."
@echo "🔍 DEBUG: Checking Go installation..."
@echo "DEBUG: Checking Go installation..."
@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
@echo "🔍 DEBUG: Go version: $$(go version)"
@echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)..."
@echo "DEBUG: Go version: $$(go version)"
@echo "DEBUG: Checking binary at $(WEED_BINARY)..."
@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
@echo "🔍 DEBUG: Binary size: $$(ls -lh $(WEED_BINARY) | awk '{print $$5}')"
@echo "🔍 DEBUG: Binary permissions: $$(ls -la $(WEED_BINARY) | awk '{print $$1}')"
@echo "🔍 DEBUG: Checking Go module dependencies..."
@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1)
@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1)
@echo "✅ All dependencies are available"
@echo "OK All dependencies are available"
# Start SeaweedFS server for testing
start-server: check-deps
@@ -77,23 +78,26 @@ start-server: check-deps
@echo "🔍 DEBUG: Checking config file at ../../../docker/compose/s3.json"
@ls -la ../../../docker/compose/s3.json || echo "⚠️ Config file not found, continuing without it"
@echo "🔍 DEBUG: Creating volume directory..."
@mkdir -p ./test-volume-data
@echo "🔍 DEBUG: Launching SeaweedFS server in background..."
@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
@$(WEED_BINARY) server \
-debug \
-s3 \
@mkdir -p $(SERVER_DIR)
@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
@$(WEED_BINARY) mini \
-dir=$(SERVER_DIR) \
-s3.port=$(S3_PORT) \
-s3.allowDeleteBucketNotEmpty=true \
-s3.config=../../../docker/compose/s3.json \
-filer \
-filer.maxMB=64 \
-master.volumeSizeLimitMB=50 \
-volume.max=100 \
-dir=./test-volume-data \
-volume.preStopSeconds=1 \
-metricsPort=9324 \
> weed-test.log 2>&1 & echo $$! > weed-server.pid
-s3.config=$(S3_CONFIG) \
> weed-test.log 2>&1 & \
echo $$! > weed-test.pid
@echo "Waiting for S3 server to be ready..."
@for i in $$(seq 1 30); do \
if echo | nc -z localhost $(S3_PORT); then \
echo "S3 server is ready!"; \
exit 0; \
fi; \
sleep 1; \
done; \
echo "S3 server failed to start"; \
exit 1 > weed-server.pid
@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
@echo "🔍 DEBUG: Checking if PID is still running..."
@sleep 2
test/s3/delete/s3_multi_delete_versioning_test.go
+164
View File
@@ -0,0 +1,164 @@
package delete
import (
"bytes"
"context"
"fmt"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
testEndpoint = "http://localhost:8333"
testAccessKey = "admin"
testSecretKey = "admin"
testRegion = "us-east-1"
)
func getTestClient(t *testing.T) *s3.Client {
cfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithRegion(testRegion),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
testAccessKey,
testSecretKey,
"",
)),
)
require.NoError(t, err)
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(testEndpoint)
o.UsePathStyle = true
})
return client
}
func createTestBucket(t *testing.T, client *s3.Client) string {
bucketName := fmt.Sprintf("test-multi-delete-%d", time.Now().UnixNano())
_, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
_, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
Bucket: aws.String(bucketName),
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
require.NoError(t, err)
return bucketName
}
func cleanupBucket(t *testing.T, client *s3.Client, bucket string) {
listResp, _ := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
})
if listResp != nil {
var objectsToDelete []types.ObjectIdentifier
for _, version := range listResp.Versions {
objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
Key: version.Key,
VersionId: version.VersionId,
})
}
for _, marker := range listResp.DeleteMarkers {
objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
Key: marker.Key,
VersionId: marker.VersionId,
})
}
if len(objectsToDelete) > 0 {
_, _ = client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
Bucket: aws.String(bucket),
Delete: &types.Delete{
Objects: objectsToDelete,
Quiet: aws.Bool(false),
},
})
}
}
_, _ = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
Bucket: aws.String(bucket),
})
}
func TestVersioningMultiObjectDelete(t *testing.T) {
client := getTestClient(t)
bucket := createTestBucket(t, client)
defer cleanupBucket(t, client, bucket)
key := "key"
numVersions := 2
var versionIds []string
for i := 0; i < numVersions; i++ {
content := fmt.Sprintf("content-%d", i)
putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: bytes.NewReader([]byte(content)),
})
require.NoError(t, err)
require.NotNil(t, putResp.VersionId)
versionIds = append(versionIds, *putResp.VersionId)
}
assert.Len(t, versionIds, 2)
var objectsToDelete []types.ObjectIdentifier
for _, vid := range versionIds {
objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{
Key: aws.String(key),
VersionId: aws.String(vid),
})
}
deleteResp, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
Bucket: aws.String(bucket),
Delete: &types.Delete{
Objects: objectsToDelete,
},
})
require.NoError(t, err)
t.Logf("Delete response: Deleted=%d, Errors=%d", len(deleteResp.Deleted), len(deleteResp.Errors))
listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
})
require.NoError(t, err)
if listResp.Versions != nil && len(listResp.Versions) > 0 {
t.Errorf("FAIL: Expected no versions, but found %d versions:", len(listResp.Versions))
for _, v := range listResp.Versions {
t.Logf(" - Key=%s, VersionId=%s, IsLatest=%v", *v.Key, *v.VersionId, v.IsLatest)
}
} else {
t.Logf("PASS: Versions correctly deleted")
}
assert.Nil(t, listResp.Versions, "Expected no versions after deletion")
deleteResp2, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{
Bucket: aws.String(bucket),
Delete: &types.Delete{
Objects: objectsToDelete,
},
})
require.NoError(t, err)
assert.Empty(t, deleteResp2.Errors, "Idempotent delete should not return errors")
}
test/s3/filer_group/Makefile
+25 -34
View File
@@ -16,6 +16,8 @@ TEST_PATTERN := Test
# Filer group configuration
FILER_GROUP := testgroup
SERVER_DIR := ./test-volume-data
S3_CONFIG := ../../../docker/compose/s3.json
# Default target
help:
@@ -44,7 +46,7 @@ build-weed:
@echo "Building SeaweedFS binary..."
@cd ../../../weed && go build -o weed_binary .
@chmod +x $(WEED_BINARY)
@echo "✅ SeaweedFS binary built at $(WEED_BINARY)"
@echo "OK SeaweedFS binary built at $(WEED_BINARY)"
check-deps: build-weed
@echo "Checking dependencies..."
@@ -53,51 +55,40 @@ check-deps: build-weed
@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1)
@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1)
@echo "✅ All dependencies are available"
@echo "OK All dependencies are available"
# Start SeaweedFS server with filer group configured
start-server: check-deps
@echo "Starting SeaweedFS server with filer group: $(FILER_GROUP)..."
@rm -f weed-server.pid
@mkdir -p ./test-volume-data
@mkdir -p $(SERVER_DIR)
@if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \
echo "⚠️ Port $(S3_PORT) is already in use"; \
echo "WARNING: Port $(S3_PORT) is already in use"; \
exit 1; \
fi
@echo "Launching SeaweedFS server with filer group $(FILER_GROUP)..."
@$(WEED_BINARY) server \
@export AWS_ACCESS_KEY_ID=some_access_key1 && \
export AWS_SECRET_ACCESS_KEY=some_secret_key1 && \
$(WEED_BINARY) mini \
-debug \
-s3 \
-dir=$(SERVER_DIR) \
-s3.port=$(S3_PORT) \
-s3.allowDeleteBucketNotEmpty=true \
-s3.config=../../../docker/compose/s3.json \
-filer \
-filer.maxMB=64 \
-s3.config=$(S3_CONFIG) \
-filer.filerGroup=$(FILER_GROUP) \
-master.volumeSizeLimitMB=50 \
-volume.max=100 \
-dir=./test-volume-data \
-volume.preStopSeconds=1 \
-metricsPort=9325 \
> weed-test.log 2>&1 & echo $$! > weed-server.pid
@echo "Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
@echo "Waiting for server to start (up to 90 seconds)..."
@for i in $$(seq 1 90); do \
if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
echo "✅ SeaweedFS server started successfully on port $(S3_PORT) with filer group $(FILER_GROUP)"; \
> weed-server.log 2>&1 & \
echo $$! > weed-server.pid
@echo "Waiting for S3 server to be ready..."
@for i in $$(seq 1 30); do \
if echo | nc -z localhost $(S3_PORT); then \
echo "S3 server is ready!"; \
exit 0; \
fi; \
if [ $$i -eq 30 ]; then \
echo "⚠️ Server taking longer than expected (30s), checking logs..."; \
if [ -f weed-test.log ]; then \
tail -20 weed-test.log; \
fi; \
fi; \
sleep 1; \
done; \
echo "❌ Server failed to start within 90 seconds"; \
if [ -f weed-test.log ]; then \
cat weed-test.log; \
echo "❌ Server failed to start within 30 seconds"; \
if [ -f weed-server.log ]; then \
cat weed-server.log; \
fi; \
exit 1
@@ -126,9 +117,9 @@ stop-server:
# Show server logs
logs:
@if test -f weed-test.log; then \
@if test -f weed-server.log; then \
echo "=== SeaweedFS Server Logs ==="; \
tail -f weed-test.log; \
tail -f weed-server.log; \
else \
echo "No log file found. Server may not be running."; \
fi
@@ -146,7 +137,7 @@ test-with-server: start-server
@echo "Test pattern: $(TEST_PATTERN)"
@echo "Test timeout: $(TEST_TIMEOUT)"
@trap "$(MAKE) stop-server" EXIT; \
$(MAKE) test || (echo "❌ Tests failed, showing server logs:" && echo "=== Last 50 lines of server logs ===" && tail -50 weed-test.log && echo "=== End of server logs ===" && exit 1)
$(MAKE) test || (echo "❌ Tests failed, showing server logs:" && echo "=== Last 50 lines of server logs ===" && tail -50 weed-server.log && echo "=== End of server logs ===" && exit 1)
@$(MAKE) stop-server
@echo "✅ Tests completed and server stopped"
@@ -154,7 +145,7 @@ test-with-server: start-server
clean:
@echo "Cleaning up test artifacts..."
@$(MAKE) stop-server
@rm -f weed-test*.log weed-server.pid
@rm -f weed-server.log weed-test*.log weed-server.pid
@rm -rf test-volume-data/
@go clean -testcache
@echo "✅ Cleanup completed"
test/s3/filer_group/test_config.json
-1
View File
@@ -1,6 +1,5 @@
{
"s3_endpoint": "http://localhost:8333",
"master_address": "localhost:9333",
"access_key": "some_access_key1",
"secret_key": "some_secret_key1",
"region": "us-east-1",
test/s3/iam/Makefile
+37 -62
View File
@@ -19,6 +19,7 @@ MASTER_PID_FILE = /tmp/weed-master.pid
VOLUME_PID_FILE = /tmp/weed-volume.pid
FILER_PID_FILE = /tmp/weed-filer.pid
S3_PID_FILE = /tmp/weed-s3.pid
MINI_PID_FILE = /tmp/weed-mini.pid
help: ## Show this help message
@echo "SeaweedFS S3 IAM Integration Tests"
@@ -49,80 +50,54 @@ test: clean setup start-services run-tests stop-services ## Run complete IAM int
test-quick: run-tests ## Run tests assuming services are already running
run-tests: ## Execute the Go tests
@echo "🧪 Running S3 IAM Integration Tests..."
@echo "Running S3 IAM Integration Tests..."
go test -v -timeout $(TEST_TIMEOUT) ./...
setup: ## Setup test environment
@echo "🔧 Setting up test environment..."
@echo "Setting up test environment..."
@mkdir -p test-volume-data/filerldb2
@mkdir -p test-volume-data/m9333
start-services: ## Start SeaweedFS services for testing
@echo "🚀 Starting SeaweedFS services..."
@echo "Starting master server..."
@$(WEED_BINARY) master -port=$(MASTER_PORT) \
-mdir=test-volume-data/m9333 \
-peers=none > weed-master.log 2>&1 & \
echo $$! > $(MASTER_PID_FILE)
@echo "Waiting for master server to be ready..."
@timeout 60 bash -c 'until curl -s http://localhost:$(MASTER_PORT)/cluster/status > /dev/null 2>&1; do echo "Waiting for master server..."; sleep 2; done' || (echo "❌ Master failed to start, checking logs..." && tail -20 weed-master.log && exit 1)
@echo "✅ Master server is ready"
@echo "Starting volume server..."
@$(WEED_BINARY) volume -port=$(VOLUME_PORT) \
-ip=localhost \
-dataCenter=dc1 -rack=rack1 \
@echo "Starting SeaweedFS services using weed mini..."
@mkdir -p test-volume-data
@$(WEED_BINARY) mini \
-dir=test-volume-data \
-max=100 \
-master=localhost:$(MASTER_PORT) > weed-volume.log 2>&1 & \
echo $$! > $(VOLUME_PID_FILE)
-s3.port=$(S3_PORT) \
-s3.config=test_config.json \
-s3.iam.config=$(CURDIR)/iam_config.json \
> weed-mini.log 2>&1 & \
echo $$! > $(MINI_PID_FILE)
@echo "Waiting for volume server to be ready..."
@timeout 60 bash -c 'until curl -s http://localhost:$(VOLUME_PORT)/status > /dev/null 2>&1; do echo "Waiting for volume server..."; sleep 2; done' || (echo "❌ Volume server failed to start, checking logs..." && tail -20 weed-volume.log && exit 1)
@echo "✅ Volume server is ready"
@echo "Starting filer server..."
@$(WEED_BINARY) filer -port=$(FILER_PORT) \
-defaultStoreDir=test-volume-data/filerldb2 \
-master=localhost:$(MASTER_PORT) > weed-filer.log 2>&1 & \
echo $$! > $(FILER_PID_FILE)
@echo "Waiting for filer server to be ready..."
@timeout 60 bash -c 'until curl -s http://localhost:$(FILER_PORT)/status > /dev/null 2>&1; do echo "Waiting for filer server..."; sleep 2; done' || (echo "❌ Filer failed to start, checking logs..." && tail -20 weed-filer.log && exit 1)
@echo "✅ Filer server is ready"
@echo "Starting S3 API server with IAM..."
@$(WEED_BINARY) -v=3 s3 -port=$(S3_PORT) \
-filer=localhost:$(FILER_PORT) \
-config=test_config.json \
-iam.config=$(CURDIR)/iam_config.json > weed-s3.log 2>&1 & \
echo $$! > $(S3_PID_FILE)
@echo "Waiting for S3 API server to be ready..."
@timeout 60 bash -c 'until curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1; do echo "Waiting for S3 API server..."; sleep 2; done' || (echo "❌ S3 API failed to start, checking logs..." && tail -20 weed-s3.log && exit 1)
@echo "✅ S3 API server is ready"
@echo "✅ All services started and ready"
@echo "Waiting for services to be ready..."
@$(MAKE) wait-for-services
@echo "OK All services started and ready"
wait-for-services: ## Wait for all services to be ready
@echo "⏳ Waiting for services to be ready..."
@echo "Checking master server..."
@timeout 30 bash -c 'until curl -s http://localhost:$(MASTER_PORT)/cluster/status > /dev/null; do sleep 1; done' || (echo "❌ Master failed to start" && exit 1)
@echo "Checking filer server..."
@timeout 30 bash -c 'until curl -s http://localhost:$(FILER_PORT)/status > /dev/null; do sleep 1; done' || (echo "❌ Filer failed to start" && exit 1)
@echo "Waiting for services to be ready..."
@echo "Checking S3 API server..."
@timeout 30 bash -c 'until curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1; do sleep 1; done' || (echo "❌ S3 API failed to start" && exit 1)
@for i in $$(seq 1 30); do \
if curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1; then \
echo "OK S3 API server is ready"; \
exit 0; \
fi; \
sleep 1; \
done; \
echo "ERROR S3 API failed to start"; \
exit 1
@echo "Pre-allocating volumes for concurrent operations..."
@curl -s "http://localhost:$(MASTER_PORT)/vol/grow?collection=default&count=10&replication=000" > /dev/null || echo "⚠️ Volume pre-allocation failed, but continuing..."
@curl -s "http://localhost:$(MASTER_PORT)/vol/grow?collection=default&count=10&replication=000" > /dev/null || echo "WARNING Volume pre-allocation failed, but continuing..."
@sleep 3
@echo "✅ All services are ready"
@echo "OK All services are ready"
stop-services: ## Stop all SeaweedFS services
@echo "🛑 Stopping SeaweedFS services..."
@echo "Stopping SeaweedFS services..."
@if [ -f $(MINI_PID_FILE) ]; then \
echo "Stopping weed mini..."; \
kill $$(cat $(MINI_PID_FILE)) 2>/dev/null || true; \
rm -f $(MINI_PID_FILE); \
fi
@if [ -f $(S3_PID_FILE) ]; then \
echo "Stopping S3 API server..."; \
kill $$(cat $(S3_PID_FILE)) 2>/dev/null || true; \
@@ -143,17 +118,17 @@ stop-services: ## Stop all SeaweedFS services
kill $$(cat $(MASTER_PID_FILE)) 2>/dev/null || true; \
rm -f $(MASTER_PID_FILE); \
fi
@echo "✅ All services stopped"
@echo "OK All services stopped"
clean: stop-services ## Clean up test environment
@echo "🧹 Cleaning up test environment..."
@echo "Cleaning up test environment..."
@rm -rf test-volume-data
@rm -f weed-*.log
@rm -f *.test
@echo "✅ Cleanup complete"
@echo "Cleanup complete"
logs: ## Show service logs
@echo "📋 Service Logs:"
@echo "Service Logs:"
@echo "=== Master Log ==="
@tail -20 weed-master.log 2>/dev/null || echo "No master log"
@echo ""
@@ -167,7 +142,7 @@ logs: ## Show service logs
@tail -20 weed-s3.log 2>/dev/null || echo "No S3 log"
status: ## Check service status
@echo "📊 Service Status:"
@echo "Service Status:"
@echo -n "Master: "; curl -s http://localhost:$(MASTER_PORT)/cluster/status > /dev/null 2>&1 && echo "✅ Running" || echo "❌ Not running"
@echo -n "Filer: "; curl -s http://localhost:$(FILER_PORT)/status > /dev/null 2>&1 && echo "✅ Running" || echo "❌ Not running"
@echo -n "S3 API: "; curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1 && echo "✅ Running" || echo "❌ Not running"
test/s3/iam/s3_service_account_security_test.go
+431
View File
@@ -0,0 +1,431 @@
package iam
// Integration tests for SeaweedFS service accounts.
// These tests ensure comprehensive coverage of service account functionality
// including security, access control, and expiration.
import (
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestServiceAccountS3Access verifies that service accounts can actually
// perform S3 operations using their credentials.
func TestServiceAccountS3Access(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !isSeaweedFSRunning(t) {
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
}
// Setup: Create a parent user
parentUserName := fmt.Sprintf("s3access-test-%d", time.Now().UnixNano())
// Create parent user
resp, err := callIAMAPI(t, "CreateUser", url.Values{
"UserName": {parentUserName},
})
require.NoError(t, err)
resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode, "Failed to create parent user")
defer func() {
// Cleanup: delete parent user
callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}})
}()
// Create service account for the parent user
createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
"ParentUser": {parentUserName},
"Description": {"S3 Access Test Service Account"},
})
require.NoError(t, err)
defer createResp.Body.Close()
require.Equal(t, http.StatusOK, createResp.StatusCode, "Failed to create service account")
body, err := io.ReadAll(createResp.Body)
require.NoError(t, err)
var saResp CreateServiceAccountResponse
err = xml.Unmarshal(body, &saResp)
require.NoError(t, err, "Failed to parse CreateServiceAccount response: %s", string(body))
accessKeyId := saResp.CreateServiceAccountResult.ServiceAccount.AccessKeyId
secretAccessKey := saResp.CreateServiceAccountResult.ServiceAccount.SecretAccessKey
saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId
require.NotEmpty(t, accessKeyId, "AccessKeyId should not be empty")
require.NotEmpty(t, secretAccessKey, "SecretAccessKey should not be empty")
defer func() {
// Cleanup: delete service account
callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}})
}()
t.Run("list_buckets_with_sa_credentials", func(t *testing.T) {
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(TestIAMEndpoint),
Credentials: credentials.NewStaticCredentials(
accessKeyId,
secretAccessKey,
"",
),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
})
require.NoError(t, err)
s3Client := s3.New(sess)
_, err = s3Client.ListBuckets(&s3.ListBucketsInput{})
// We don't necessarily expect success (depends on permissions),
// but we should NOT get InvalidAccessKeyId or SignatureDoesNotMatch
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
assert.NotEqual(t, "InvalidAccessKeyId", aerr.Code(),
"Service account credentials should be recognized")
assert.NotEqual(t, "SignatureDoesNotMatch", aerr.Code(),
"Service account signature should be valid")
}
}
})
t.Run("create_bucket_with_sa_credentials", func(t *testing.T) {
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(TestIAMEndpoint),
Credentials: credentials.NewStaticCredentials(
accessKeyId,
secretAccessKey,
"",
),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
})
require.NoError(t, err)
s3Client := s3.New(sess)
bucketName := fmt.Sprintf("sa-test-bucket-%d", time.Now().UnixNano())
_, err = s3Client.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucketName),
})
// Check that we get a proper response (success or AccessDenied based on policy)
// but NOT InvalidAccessKeyId
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
assert.NotEqual(t, "InvalidAccessKeyId", aerr.Code(),
"Service account credentials should be recognized")
assert.NotEqual(t, "SignatureDoesNotMatch", aerr.Code(),
"Service account signature should be valid")
}
} else {
// Cleanup if bucket was created
defer s3Client.DeleteBucket(&s3.DeleteBucketInput{
Bucket: aws.String(bucketName),
})
}
})
}
// TestServiceAccountExpiration verifies that expired service accounts
// are properly rejected.
func TestServiceAccountExpiration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !isSeaweedFSRunning(t) {
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
}
// Setup: Create a parent user
parentUserName := fmt.Sprintf("expiry-test-%d", time.Now().UnixNano())
resp, err := callIAMAPI(t, "CreateUser", url.Values{
"UserName": {parentUserName},
})
require.NoError(t, err)
resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
defer func() {
callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}})
}()
t.Run("reject_past_expiration", func(t *testing.T) {
// Try to create a service account with expiration in the past
pastExpiration := time.Now().Add(-1 * time.Hour).Unix()
createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
"ParentUser": {parentUserName},
"Description": {"Should fail - past expiration"},
"Expiration": {strconv.FormatInt(pastExpiration, 10)},
})
require.NoError(t, err)
defer createResp.Body.Close()
// Should fail because expiration is in the past
assert.NotEqual(t, http.StatusOK, createResp.StatusCode,
"Creating service account with past expiration should fail")
})
t.Run("accept_future_expiration", func(t *testing.T) {
// Create a service account with expiration in the future
futureExpiration := time.Now().Add(24 * time.Hour).Unix()
createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
"ParentUser": {parentUserName},
"Description": {"Should succeed - future expiration"},
"Expiration": {strconv.FormatInt(futureExpiration, 10)},
})
require.NoError(t, err)
defer createResp.Body.Close()
assert.Equal(t, http.StatusOK, createResp.StatusCode,
"Creating service account with future expiration should succeed")
// Parse response to get service account ID for cleanup
if createResp.StatusCode == http.StatusOK {
body, err := io.ReadAll(createResp.Body)
require.NoError(t, err)
var saResp CreateServiceAccountResponse
require.NoError(t, xml.Unmarshal(body, &saResp))
saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId
if saId != "" {
t.Cleanup(func() {
callIAMAPI(t, "DeleteServiceAccount", url.Values{
"ServiceAccountId": {saId},
})
})
}
}
})
t.Run("reject_past_expiration_on_update", func(t *testing.T) {
// Create a valid service account first
futureExpiration := time.Now().Add(24 * time.Hour).Unix()
createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
"ParentUser": {parentUserName},
"Description": {"For update test"},
"Expiration": {strconv.FormatInt(futureExpiration, 10)},
})
require.NoError(t, err)
defer createResp.Body.Close()
require.Equal(t, http.StatusOK, createResp.StatusCode)
body, err := io.ReadAll(createResp.Body)
require.NoError(t, err)
var saResp CreateServiceAccountResponse
err = xml.Unmarshal(body, &saResp)
require.NoError(t, err)
saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId
require.NotEmpty(t, saId)
defer func() {
callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}})
}()
// Try to update with past expiration
pastExpiration := time.Now().Add(-1 * time.Hour).Unix()
updateResp, err := callIAMAPI(t, "UpdateServiceAccount", url.Values{
"ServiceAccountId": {saId},
"Expiration": {strconv.FormatInt(pastExpiration, 10)},
})
require.NoError(t, err)
defer updateResp.Body.Close()
// Should fail because expiration is in the past
assert.NotEqual(t, http.StatusOK, updateResp.StatusCode,
"Updating service account with past expiration should fail")
})
}
// TestServiceAccountInheritedPermissions verifies that service accounts
// inherit their parent user's permissions.
// This is a key security test - SAs should not have MORE permissions than parent.
func TestServiceAccountInheritedPermissions(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !isSeaweedFSRunning(t) {
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
}
// Setup: Create a parent user
parentUserName := fmt.Sprintf("inherit-test-%d", time.Now().UnixNano())
resp, err := callIAMAPI(t, "CreateUser", url.Values{
"UserName": {parentUserName},
})
require.NoError(t, err)
resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
defer func() {
callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}})
}()
// Create service account
createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
"ParentUser": {parentUserName},
"Description": {"Permissions inheritance test"},
})
require.NoError(t, err)
defer createResp.Body.Close()
require.Equal(t, http.StatusOK, createResp.StatusCode)
body, err := io.ReadAll(createResp.Body)
require.NoError(t, err)
var saResp CreateServiceAccountResponse
err = xml.Unmarshal(body, &saResp)
require.NoError(t, err)
saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId
require.NotEmpty(t, saId)
defer func() {
callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}})
}()
t.Run("service_account_linked_to_parent", func(t *testing.T) {
// Verify the service account is correctly linked to the parent
getResp, err := callIAMAPI(t, "GetServiceAccount", url.Values{
"ServiceAccountId": {saId},
})
require.NoError(t, err)
defer getResp.Body.Close()
body, err := io.ReadAll(getResp.Body)
require.NoError(t, err)
var result GetServiceAccountResponse
err = xml.Unmarshal(body, &result)
require.NoError(t, err, "Failed to parse response: %s", string(body))
assert.Equal(t, parentUserName, result.GetServiceAccountResult.ServiceAccount.ParentUser,
"Service account should be linked to correct parent user")
})
t.Run("list_shows_correct_parent", func(t *testing.T) {
// List service accounts filtered by parent
listResp, err := callIAMAPI(t, "ListServiceAccounts", url.Values{
"ParentUser": {parentUserName},
})
require.NoError(t, err)
defer listResp.Body.Close()
body, err := io.ReadAll(listResp.Body)
require.NoError(t, err)
var listResult ListServiceAccountsResponse
err = xml.Unmarshal(body, &listResult)
require.NoError(t, err)
// Should find at least one service account for this parent
found := false
for _, sa := range listResult.ListServiceAccountsResult.ServiceAccounts {
if sa.ServiceAccountId == saId {
found = true
assert.Equal(t, parentUserName, sa.ParentUser)
break
}
}
assert.True(t, found, "Service account should appear in list filtered by parent")
})
}
// TestServiceAccountAccessKeyFormat verifies that service account access keys
// follow the correct AWS format.
func TestServiceAccountAccessKeyFormat(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !isSeaweedFSRunning(t) {
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
}
// Setup: Create a parent user
parentUserName := fmt.Sprintf("keyformat-test-%d", time.Now().UnixNano())
resp, err := callIAMAPI(t, "CreateUser", url.Values{
"UserName": {parentUserName},
})
require.NoError(t, err)
resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
defer func() {
callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}})
}()
createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
"ParentUser": {parentUserName},
"Description": {"Key format test"},
})
require.NoError(t, err)
defer createResp.Body.Close()
require.Equal(t, http.StatusOK, createResp.StatusCode)
body, err := io.ReadAll(createResp.Body)
require.NoError(t, err)
var saResp CreateServiceAccountResponse
err = xml.Unmarshal(body, &saResp)
require.NoError(t, err)
accessKeyId := saResp.CreateServiceAccountResult.ServiceAccount.AccessKeyId
secretAccessKey := saResp.CreateServiceAccountResult.ServiceAccount.SecretAccessKey
saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId
defer func() {
callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}})
}()
t.Run("access_key_has_correct_prefix", func(t *testing.T) {
// Service account access keys should start with ABIA
assert.True(t, len(accessKeyId) >= 4,
"Access key should be at least 4 characters")
assert.Equal(t, "ABIA", accessKeyId[:4],
"Service account access key should start with ABIA prefix")
})
t.Run("access_key_has_correct_length", func(t *testing.T) {
// AWS access keys are 20 characters
assert.Equal(t, 20, len(accessKeyId),
"Access key should be exactly 20 characters (AWS standard)")
})
t.Run("secret_key_has_correct_length", func(t *testing.T) {
// AWS secret keys are 40 characters
assert.Equal(t, 40, len(secretAccessKey),
"Secret key should be exactly 40 characters (AWS standard)")
})
}
test/s3/iam/s3_service_account_test.go
+363
View File
@@ -0,0 +1,363 @@
package iam
import (
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// Service Account API test constants
const (
TestIAMEndpoint = "http://localhost:8333"
)
// ServiceAccountInfo represents the response structure for service account operations
type ServiceAccountInfo struct {
ServiceAccountId string `xml:"ServiceAccountId"`
ParentUser string `xml:"ParentUser"`
Description string `xml:"Description,omitempty"`
AccessKeyId string `xml:"AccessKeyId"`
SecretAccessKey string `xml:"SecretAccessKey,omitempty"`
Status string `xml:"Status"`
Expiration string `xml:"Expiration,omitempty"`
CreateDate string `xml:"CreateDate"`
}
// CreateServiceAccountResponse represents the response for CreateServiceAccount
type CreateServiceAccountResponse struct {
XMLName xml.Name `xml:"CreateServiceAccountResponse"`
CreateServiceAccountResult struct {
ServiceAccount ServiceAccountInfo `xml:"ServiceAccount"`
} `xml:"CreateServiceAccountResult"`
}
// ListServiceAccountsResponse represents the response for ListServiceAccounts
type ListServiceAccountsResponse struct {
XMLName xml.Name `xml:"ListServiceAccountsResponse"`
ListServiceAccountsResult struct {
ServiceAccounts []ServiceAccountInfo `xml:"ServiceAccounts>member"`
IsTruncated bool `xml:"IsTruncated"`
} `xml:"ListServiceAccountsResult"`
}
// GetServiceAccountResponse represents the response for GetServiceAccount
type GetServiceAccountResponse struct {
XMLName xml.Name `xml:"GetServiceAccountResponse"`
GetServiceAccountResult struct {
ServiceAccount ServiceAccountInfo `xml:"ServiceAccount"`
} `xml:"GetServiceAccountResult"`
}
// TestServiceAccountLifecycle tests the complete lifecycle of service accounts
// This is a high-value test covering Create, Get, List, Update, Delete operations
func TestServiceAccountLifecycle(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
// Check if SeaweedFS is running
if !isSeaweedFSRunning(t) {
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
}
// First, ensure the parent user exists
parentUserName := fmt.Sprintf("testuser-%d", time.Now().UnixNano())
t.Run("create_parent_user", func(t *testing.T) {
resp, err := callIAMAPI(t, "CreateUser", url.Values{
"UserName": {parentUserName},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode, "CreateUser should succeed")
})
// Store service account IDs for cleanup
var createdServiceAccounts []string
defer func() {
// Cleanup: delete all service accounts first, then parent user
for _, saId := range createdServiceAccounts {
resp, _ := callIAMAPI(t, "DeleteServiceAccount", url.Values{
"ServiceAccountId": {saId},
})
if resp != nil {
resp.Body.Close()
}
}
// Now delete the parent user
resp, _ := callIAMAPI(t, "DeleteUser", url.Values{
"UserName": {parentUserName},
})
if resp != nil {
resp.Body.Close()
}
}()
var createdSAId string
var createdAccessKeyId string
var createdSecretAccessKey string
t.Run("create_service_account", func(t *testing.T) {
resp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
"ParentUser": {parentUserName},
"Description": {"Test service account for CI/CD"},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode, "CreateServiceAccount should succeed")
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
var createResp CreateServiceAccountResponse
err = xml.Unmarshal(body, &createResp)
require.NoError(t, err)
sa := createResp.CreateServiceAccountResult.ServiceAccount
createdSAId = sa.ServiceAccountId
createdAccessKeyId = sa.AccessKeyId
createdSecretAccessKey = sa.SecretAccessKey
// Add to cleanup list
createdServiceAccounts = append(createdServiceAccounts, createdSAId)
assert.NotEmpty(t, createdSAId, "ServiceAccountId should not be empty")
assert.Equal(t, parentUserName, sa.ParentUser, "ParentUser should match")
assert.Equal(t, "Test service account for CI/CD", sa.Description)
assert.Equal(t, "Active", sa.Status)
assert.NotEmpty(t, sa.AccessKeyId, "AccessKeyId should not be empty")
assert.NotEmpty(t, sa.SecretAccessKey, "SecretAccessKey should be returned on create")
assert.True(t, strings.HasPrefix(sa.AccessKeyId, "ABIA"),
"Service account AccessKeyId should have ABIA prefix")
t.Logf("Created service account: ID=%s, AccessKeyId=%s", createdSAId, createdAccessKeyId)
})
t.Run("get_service_account", func(t *testing.T) {
require.NotEmpty(t, createdSAId, "Service account should have been created")
resp, err := callIAMAPI(t, "GetServiceAccount", url.Values{
"ServiceAccountId": {createdSAId},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
var getResp GetServiceAccountResponse
err = xml.Unmarshal(body, &getResp)
require.NoError(t, err)
sa := getResp.GetServiceAccountResult.ServiceAccount
assert.Equal(t, createdSAId, sa.ServiceAccountId)
assert.Equal(t, parentUserName, sa.ParentUser)
assert.Empty(t, sa.SecretAccessKey, "SecretAccessKey should not be returned on Get")
})
t.Run("list_service_accounts", func(t *testing.T) {
resp, err := callIAMAPI(t, "ListServiceAccounts", url.Values{
"ParentUser": {parentUserName},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
var listResp ListServiceAccountsResponse
err = xml.Unmarshal(body, &listResp)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(listResp.ListServiceAccountsResult.ServiceAccounts), 1,
"Should have at least one service account for the parent user")
})
t.Run("update_service_account_status", func(t *testing.T) {
require.NotEmpty(t, createdSAId)
// Disable the service account
resp, err := callIAMAPI(t, "UpdateServiceAccount", url.Values{
"ServiceAccountId": {createdSAId},
"Status": {"Inactive"},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
// Verify it's now inactive
getResp, err := callIAMAPI(t, "GetServiceAccount", url.Values{
"ServiceAccountId": {createdSAId},
})
require.NoError(t, err)
defer getResp.Body.Close()
body, err := io.ReadAll(getResp.Body)
require.NoError(t, err)
var result GetServiceAccountResponse
err = xml.Unmarshal(body, &result)
require.NoError(t, err, "Failed to parse response: %s", string(body))
assert.Equal(t, "Inactive", result.GetServiceAccountResult.ServiceAccount.Status)
})
// Test that credentials could be used (verify they work with AWS SDK)
// This must run BEFORE delete_service_account to use valid credentials
t.Run("use_service_account_credentials", func(t *testing.T) {
require.NotEmpty(t, createdAccessKeyId)
require.NotEmpty(t, createdSecretAccessKey)
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String(TestIAMEndpoint), // IAM and S3 usually on same port in mini-seaweed
Credentials: credentials.NewStaticCredentials(
createdAccessKeyId,
createdSecretAccessKey,
"",
),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
})
require.NoError(t, err)
s3Client := s3.New(sess)
_, err = s3Client.ListBuckets(&s3.ListBucketsInput{})
// Note: we don't necessarily expect success if no buckets/permissions
// but we expect it not to fail with "InvalidAccessKeyId" or "SignatureDoesNotMatch"
if err != nil {
if aerr, ok := err.(awserr.Error); ok {
assert.NotEqual(t, "InvalidAccessKeyId", aerr.Code(), "Credentials should be valid")
assert.NotEqual(t, "SignatureDoesNotMatch", aerr.Code(), "Signature should be valid")
}
}
})
t.Run("delete_service_account", func(t *testing.T) {
require.NotEmpty(t, createdSAId)
resp, err := callIAMAPI(t, "DeleteServiceAccount", url.Values{
"ServiceAccountId": {createdSAId},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.Equal(t, http.StatusOK, resp.StatusCode)
// Verify it no longer exists
getResp, err := callIAMAPI(t, "GetServiceAccount", url.Values{
"ServiceAccountId": {createdSAId},
})
require.NoError(t, err)
defer getResp.Body.Close()
// Should return an error (not found)
assert.NotEqual(t, http.StatusOK, getResp.StatusCode,
"GetServiceAccount should fail after deletion")
})
}
// TestServiceAccountValidation tests validation of service account operations
func TestServiceAccountValidation(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !isSeaweedFSRunning(t) {
t.Skip("SeaweedFS is not running at", TestIAMEndpoint)
}
t.Run("create_without_parent_user", func(t *testing.T) {
resp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
"Description": {"Test without parent"},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.NotEqual(t, http.StatusOK, resp.StatusCode,
"CreateServiceAccount without ParentUser should fail")
})
t.Run("create_with_nonexistent_parent", func(t *testing.T) {
resp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{
"ParentUser": {"nonexistent-user-12345"},
"Description": {"Test with nonexistent parent"},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.NotEqual(t, http.StatusOK, resp.StatusCode,
"CreateServiceAccount with nonexistent parent should fail")
})
t.Run("get_nonexistent_service_account", func(t *testing.T) {
resp, err := callIAMAPI(t, "GetServiceAccount", url.Values{
"ServiceAccountId": {"sa-NONEXISTENT123"},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.NotEqual(t, http.StatusOK, resp.StatusCode,
"GetServiceAccount for nonexistent ID should fail")
})
t.Run("delete_nonexistent_service_account", func(t *testing.T) {
resp, err := callIAMAPI(t, "DeleteServiceAccount", url.Values{
"ServiceAccountId": {"sa-NONEXISTENT123"},
})
require.NoError(t, err)
defer resp.Body.Close()
assert.NotEqual(t, http.StatusOK, resp.StatusCode,
"DeleteServiceAccount for nonexistent ID should fail")
})
}
// callIAMAPI is a helper to make IAM API calls
func callIAMAPI(t *testing.T, action string, params url.Values) (*http.Response, error) {
params.Set("Action", action)
req, err := http.NewRequest(http.MethodPost, TestIAMEndpoint+"/",
strings.NewReader(params.Encode()))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
client := &http.Client{Timeout: 30 * time.Second}
return client.Do(req)
}
// isSeaweedFSRunning checks if SeaweedFS S3 API is running
func isSeaweedFSRunning(t *testing.T) bool {
client := &http.Client{Timeout: 2 * time.Second}
resp, err := client.Get(TestIAMEndpoint + "/status")
if err != nil {
return false
}
defer resp.Body.Close()
return resp.StatusCode == http.StatusOK
}
test/s3/iam/s3_sts_test.go
+260
View File
@@ -0,0 +1,260 @@
package iam
import (
"encoding/xml"
"io"
"net/http"
"net/url"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// STS API test constants
const (
TestSTSEndpoint = "http://localhost:8333"
)
// AssumeRoleWithWebIdentityResponse represents the STS response
type AssumeRoleWithWebIdentityTestResponse struct {
XMLName xml.Name `xml:"AssumeRoleWithWebIdentityResponse"`
Result struct {
Credentials struct {
AccessKeyId string `xml:"AccessKeyId"`
SecretAccessKey string `xml:"SecretAccessKey"`
SessionToken string `xml:"SessionToken"`
Expiration string `xml:"Expiration"`
} `xml:"Credentials"`
SubjectFromWebIdentityToken string `xml:"SubjectFromWebIdentityToken,omitempty"`
} `xml:"AssumeRoleWithWebIdentityResult"`
}
// STSErrorResponse represents an STS error response
type STSErrorTestResponse struct {
XMLName xml.Name `xml:"ErrorResponse"`
Error struct {
Type string `xml:"Type"`
Code string `xml:"Code"`
Message string `xml:"Message"`
} `xml:"Error"`
RequestId string `xml:"RequestId"`
}
// TestAssumeRoleWithWebIdentityValidation tests input validation for the STS endpoint
func TestAssumeRoleWithWebIdentityValidation(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !isSTSEndpointRunning(t) {
t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
}
t.Run("missing_web_identity_token", func(t *testing.T) {
resp, err := callSTSAPI(t, url.Values{
"Action": {"AssumeRoleWithWebIdentity"},
"RoleArn": {"arn:aws:iam::role/test-role"},
"RoleSessionName": {"test-session"},
// WebIdentityToken is missing
})
require.NoError(t, err)
defer resp.Body.Close()
assert.NotEqual(t, http.StatusOK, resp.StatusCode,
"Should fail without WebIdentityToken")
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
var errResp STSErrorTestResponse
err = xml.Unmarshal(body, &errResp)
require.NoError(t, err, "Failed to parse error response: %s", string(body))
assert.Equal(t, "MissingParameter", errResp.Error.Code)
})
t.Run("missing_role_arn", func(t *testing.T) {
resp, err := callSTSAPI(t, url.Values{
"Action": {"AssumeRoleWithWebIdentity"},
"WebIdentityToken": {"fake-jwt-token"},
"RoleSessionName": {"test-session"},
// RoleArn is missing
})
require.NoError(t, err)
defer resp.Body.Close()
assert.NotEqual(t, http.StatusOK, resp.StatusCode,
"Should fail without RoleArn")
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
var errResp STSErrorTestResponse
err = xml.Unmarshal(body, &errResp)
require.NoError(t, err, "Failed to parse error response: %s", string(body))
assert.Equal(t, "MissingParameter", errResp.Error.Code)
})
t.Run("missing_role_session_name", func(t *testing.T) {
resp, err := callSTSAPI(t, url.Values{
"Action": {"AssumeRoleWithWebIdentity"},
"WebIdentityToken": {"fake-jwt-token"},
"RoleArn": {"arn:aws:iam::role/test-role"},
// RoleSessionName is missing
})
require.NoError(t, err)
defer resp.Body.Close()
assert.NotEqual(t, http.StatusOK, resp.StatusCode,
"Should fail without RoleSessionName")
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
var errResp STSErrorTestResponse
err = xml.Unmarshal(body, &errResp)
require.NoError(t, err, "Failed to parse error response: %s", string(body))
assert.Equal(t, "MissingParameter", errResp.Error.Code)
})
t.Run("invalid_jwt_token", func(t *testing.T) {
resp, err := callSTSAPI(t, url.Values{
"Action": {"AssumeRoleWithWebIdentity"},
"WebIdentityToken": {"not-a-valid-jwt-token"},
"RoleArn": {"arn:aws:iam::role/test-role"},
"RoleSessionName": {"test-session"},
})
require.NoError(t, err)
defer resp.Body.Close()
// Should fail with AccessDenied since the JWT is invalid
assert.NotEqual(t, http.StatusOK, resp.StatusCode,
"Should fail with invalid JWT token")
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
var errResp STSErrorTestResponse
err = xml.Unmarshal(body, &errResp)
require.NoError(t, err, "Failed to parse error response: %s", string(body))
assert.Contains(t, []string{"AccessDenied", "InvalidParameterValue"}, errResp.Error.Code)
})
}
// TestAssumeRoleWithWebIdentityWithMockJWT tests the STS endpoint with mock JWTs
// This test requires the mock OIDC provider to be configured
func TestAssumeRoleWithWebIdentityWithMockJWT(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !isSTSEndpointRunning(t) {
t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint)
}
// Create a test framework to get valid JWT tokens
framework := NewS3IAMTestFramework(t)
defer framework.Cleanup()
// Generate a test JWT using the framework
testUsername := "sts-test-user"
testRole := "readonly"
// Try to get a token - use Keycloak if available, otherwise generate a mock JWT
var token string
var err error
if framework.useKeycloak {
token, err = framework.getKeycloakToken(testUsername)
} else {
// Generate a mock JWT token with 1 hour validity
token, err = framework.generateJWTToken(testUsername, testRole, time.Hour)
}
if err != nil {
t.Skipf("Unable to generate test JWT (requires mock OIDC or Keycloak): %v", err)
}
t.Run("valid_jwt_token", func(t *testing.T) {
resp, err := callSTSAPI(t, url.Values{
"Action": {"AssumeRoleWithWebIdentity"},
"WebIdentityToken": {token},
"RoleArn": {"arn:aws:iam::role/" + testRole},
"RoleSessionName": {"integration-test-session"},
})
require.NoError(t, err)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body))
// Note: This may still fail if the role/trust policy is not configured
// In that case, we just verify the error is about trust policy, not token validation
if resp.StatusCode != http.StatusOK {
var errResp STSErrorTestResponse
err = xml.Unmarshal(body, &errResp)
require.NoError(t, err, "Failed to parse error response: %s", string(body))
assert.NotEqual(t, "InvalidParameterValue", errResp.Error.Code,
"Token validation should not fail - error should be about trust policy")
} else {
var stsResp AssumeRoleWithWebIdentityTestResponse
err = xml.Unmarshal(body, &stsResp)
require.NoError(t, err, "Failed to parse response: %s", string(body))
creds := stsResp.Result.Credentials
assert.NotEmpty(t, creds.AccessKeyId, "AccessKeyId should not be empty")
assert.NotEmpty(t, creds.SecretAccessKey, "SecretAccessKey should not be empty")
assert.NotEmpty(t, creds.SessionToken, "SessionToken should not be empty")
assert.NotEmpty(t, creds.Expiration, "Expiration should not be empty")
t.Logf("Successfully obtained temporary credentials: AccessKeyId=%s", creds.AccessKeyId)
}
})
t.Run("with_duration_seconds", func(t *testing.T) {
resp, err := callSTSAPI(t, url.Values{
"Action": {"AssumeRoleWithWebIdentity"},
"WebIdentityToken": {token},
"RoleArn": {"arn:aws:iam::role/" + testRole},
"RoleSessionName": {"integration-test-session"},
"DurationSeconds": {"3600"}, // 1 hour
})
require.NoError(t, err)
defer resp.Body.Close()
// Verify the request is accepted (even if trust policy causes rejection)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
// Should not fail with InvalidParameterValue for DurationSeconds
if resp.StatusCode != http.StatusOK {
var errResp STSErrorTestResponse
err = xml.Unmarshal(body, &errResp)
require.NoError(t, err, "Failed to parse error response: %s", string(body))
assert.NotContains(t, errResp.Error.Message, "DurationSeconds",
"DurationSeconds parameter should be accepted")
}
})
}
// callSTSAPI is a helper to make STS API calls
func callSTSAPI(t *testing.T, params url.Values) (*http.Response, error) {
req, err := http.NewRequest(http.MethodPost, TestSTSEndpoint+"/",
strings.NewReader(params.Encode()))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
client := &http.Client{Timeout: 30 * time.Second}
return client.Do(req)
}
// isSTSEndpointRunning checks if SeaweedFS STS endpoint is running
func isSTSEndpointRunning(t *testing.T) bool {
client := &http.Client{Timeout: 2 * time.Second}
resp, err := client.Get(TestSTSEndpoint + "/status")
if err != nil {
return false
}
defer resp.Body.Close()
return resp.StatusCode == http.StatusOK
}
test/s3/parquet/CROSS_FILESYSTEM_COMPATIBILITY.md
-172
View File
@@ -1,172 +0,0 @@
# Cross-Filesystem Compatibility Test Results
## Overview
This document summarizes the cross-filesystem compatibility testing between **s3fs** and **PyArrow native S3 filesystem** implementations when working with SeaweedFS.
## Test Purpose
Verify that Parquet files written using one filesystem implementation (s3fs or PyArrow native S3) can be correctly read using the other implementation, confirming true file format compatibility.
## Test Methodology
### Test Matrix
The test performs the following combinations:
1. **Write with s3fs → Read with PyArrow native S3**
2. **Write with PyArrow native S3 → Read with s3fs**
For each direction, the test:
- Creates a sample PyArrow table with multiple data types (int64, string, float64, bool)
- Writes the Parquet file using one filesystem implementation
- Reads the Parquet file using the other filesystem implementation
- Verifies data integrity by comparing:
- Row counts
- Schema equality
- Data contents (after sorting by ID to handle row order differences)
### File Sizes Tested
- **Small files**: 5 rows (quick validation)
- **Large files**: 200,000 rows (multi-row-group validation)
## Test Results
### ✅ Small Files (5 rows)
| Write Method | Read Method | Result | Read Function Used |
|--------------|-------------|--------|--------------------|
| s3fs | PyArrow native S3 | ✅ PASS | pq.read_table |
| PyArrow native S3 | s3fs | ✅ PASS | pq.read_table |
**Status**: **ALL TESTS PASSED**
### Large Files (200,000 rows)
Large file testing requires adequate volume capacity in SeaweedFS. When run with default volume settings (50MB max size), tests may encounter capacity issues with the number of large test files created simultaneously.
**Recommendation**: For large file testing, increase `VOLUME_MAX_SIZE_MB` in the Makefile or run tests with `TEST_QUICK=1` for development/validation purposes.
## Key Findings
### ✅ Full Compatibility Confirmed
**Files written with s3fs and PyArrow native S3 filesystem are fully compatible and can be read by either implementation.**
This confirms that:
1. **Identical Parquet Format**: Both s3fs and PyArrow native S3 use the same underlying PyArrow library to generate Parquet files, resulting in identical file formats at the binary level.
2. **S3 API Compatibility**: SeaweedFS's S3 implementation handles both filesystem backends correctly, with proper:
- Object creation (PutObject)
- Object reading (GetObject)
- Directory handling (implicit directories)
- Multipart uploads (for larger files)
3. **Metadata Consistency**: File metadata, schemas, and data integrity are preserved across both write and read operations regardless of which filesystem implementation is used.
## Implementation Details
### Common Write Path
Both implementations use PyArrow's `pads.write_dataset()` function:
```python
# s3fs approach
fs = s3fs.S3FileSystem(...)
pads.write_dataset(table, path, format="parquet", filesystem=fs)
# PyArrow native approach
s3 = pafs.S3FileSystem(...)
pads.write_dataset(table, path, format="parquet", filesystem=s3)
```
### Multiple Read Methods Tested
The test attempts reads using multiple PyArrow methods:
- `pq.read_table()` - Direct table reading
- `pq.ParquetDataset()` - Dataset-based reading
- `pads.dataset()` - PyArrow dataset API
All methods successfully read files written by either filesystem implementation.
## Practical Implications
### For Users
1. **Flexibility**: Users can choose either s3fs or PyArrow native S3 based on their preferences:
- **s3fs**: More mature, widely used, familiar API
- **PyArrow native**: Pure PyArrow solution, fewer dependencies
2. **Interoperability**: Teams using different tools can seamlessly share Parquet datasets stored in SeaweedFS
3. **Migration**: Easy to migrate between filesystem implementations without data conversion
### For SeaweedFS
1. **S3 Compatibility**: Confirms SeaweedFS's S3 implementation is compatible with major Python data science tools
2. **Implicit Directory Handling**: The implicit directory fix works correctly for both filesystem implementations
3. **Standard Compliance**: SeaweedFS handles S3 operations in a way that's compatible with AWS S3 behavior
## Running the Tests
### Quick Test (Recommended for Development)
```bash
cd test/s3/parquet
TEST_QUICK=1 make test-cross-fs-with-server
```
### Full Test (All File Sizes)
```bash
cd test/s3/parquet
make test-cross-fs-with-server
```
### Manual Test (Assuming Server is Running)
```bash
cd test/s3/parquet
make setup-python
make start-seaweedfs-ci
# In another terminal
TEST_QUICK=1 make test-cross-fs
# Cleanup
make stop-seaweedfs-safe
```
## Environment Variables
The test supports customization through environment variables:
- `S3_ENDPOINT_URL`: S3 endpoint (default: `http://localhost:8333`)
- `S3_ACCESS_KEY`: Access key (default: `some_access_key1`)
- `S3_SECRET_KEY`: Secret key (default: `some_secret_key1`)
- `BUCKET_NAME`: Bucket name (default: `test-parquet-bucket`)
- `TEST_QUICK`: Run only small tests (default: `0`, set to `1` for quick mode)
## Conclusion
The cross-filesystem compatibility tests demonstrate that **Parquet files written via s3fs and PyArrow native S3 filesystem are completely interchangeable**. This validates that:
1. The Parquet file format is implementation-agnostic
2. SeaweedFS's S3 API correctly handles both filesystem backends
3. Users have full flexibility in choosing their preferred filesystem implementation
This compatibility is a testament to:
- PyArrow's consistent file format generation
- SeaweedFS's robust S3 API implementation
- Proper handling of S3 semantics (especially implicit directories)
---
**Test Implementation**: `test_cross_filesystem_compatibility.py`
**Last Updated**: November 21, 2024
**Status**: ✅ All critical tests passing
test/s3/parquet/FINAL_ROOT_CAUSE_ANALYSIS.md
-58
View File
@@ -1,58 +0,0 @@
# Final Root Cause Analysis
## Overview
This document provides a deep technical analysis of the s3fs compatibility issue with PyArrow Parquet datasets on SeaweedFS, and the solution implemented to resolve it.
## Root Cause
When PyArrow writes datasets using `write_dataset()`, it creates implicit directory structures by writing files without explicit directory markers. However, some S3 workflows may create 0-byte directory markers.
### The Problem
1. **PyArrow writes dataset files** without creating explicit directory objects
2. **s3fs calls HEAD** on the directory path to check if it exists
3. **If HEAD returns 200** with `Content-Length: 0`, s3fs interprets it as a file (not a directory)
4. **PyArrow fails** when trying to read, reporting "Parquet file size is 0 bytes"
### AWS S3 Behavior
AWS S3 returns **404 Not Found** for implicit directories (directories that only exist because they have children but no explicit marker object). This allows s3fs to fall back to LIST operations to detect the directory.
## The Solution
### Implementation
Modified the S3 API HEAD handler in `weed/s3api/s3api_object_handlers.go` to:
1. **Check if object ends with `/`**: Explicit directory markers return 200 as before
2. **Check if object has children**: If a 0-byte object has children in the filer, treat it as an implicit directory
3. **Return 404 for implicit directories**: This matches AWS S3 behavior and triggers s3fs's LIST fallback
### Code Changes
The fix is implemented in the `HeadObjectHandler` function with logic to:
- Detect implicit directories by checking for child entries
- Return 404 (NoSuchKey) for implicit directories
- Preserve existing behavior for explicit directory markers and regular files
## Performance Considerations
### Optimization: Child Check Cache
- Child existence checks are performed via filer LIST operations
- Results could be cached for frequently accessed paths
- Trade-off between consistency and performance
### Impact
- Minimal performance impact for normal file operations
- Slight overhead for HEAD requests on implicit directories (one additional LIST call)
- Overall improvement in PyArrow compatibility outweighs minor performance cost
## TODO
- [ ] Add detailed benchmarking results comparing before/after fix
- [ ] Document edge cases discovered during implementation
- [ ] Add architectural diagrams showing the request flow
- [ ] Document alternative solutions considered and why they were rejected
- [ ] Add performance profiling data for child existence checks
test/s3/parquet/MINIO_DIRECTORY_HANDLING.md
-70
View File
@@ -1,70 +0,0 @@
# MinIO Directory Handling Comparison
## Overview
This document compares how MinIO handles directory markers versus SeaweedFS's implementation, and explains the different approaches to S3 directory semantics.
## MinIO's Approach
MinIO handles implicit directories similarly to AWS S3:
1. **No explicit directory objects**: Directories are implicit, defined only by object key prefixes
2. **HEAD on directory returns 404**: Consistent with AWS S3 behavior
3. **LIST operations reveal directories**: Directories are discovered through delimiter-based LIST operations
4. **Automatic prefix handling**: MinIO automatically recognizes prefixes as directories
### MinIO Implementation Details
- Uses in-memory metadata for fast prefix lookups
- Optimized for LIST operations with common delimiter (`/`)
- No persistent directory objects in storage layer
- Directories "exist" as long as they contain objects
## SeaweedFS Approach
SeaweedFS uses a filer-based approach with real directory entries:
### Before the Fix
1. **Explicit directory objects**: Could create 0-byte objects as directory markers
2. **HEAD returns 200**: Even for implicit directories
3. **Caused s3fs issues**: s3fs interpreted 0-byte HEAD responses as empty files
### After the Fix
1. **Hybrid approach**: Supports both explicit markers (with `/` suffix) and implicit directories
2. **HEAD returns 404 for implicit directories**: Matches AWS S3 and MinIO behavior
3. **Filer integration**: Uses filer's directory metadata to detect implicit directories
4. **s3fs compatibility**: Triggers proper LIST fallback behavior
## Key Differences
| Aspect | MinIO | SeaweedFS (After Fix) |
|--------|-------|----------------------|
| Directory Storage | No persistent objects | Filer directory entries |
| Implicit Directory HEAD | 404 Not Found | 404 Not Found |
| Explicit Marker HEAD | Not applicable | 200 OK (with `/` suffix) |
| Child Detection | Prefix scan | Filer LIST operation |
| Performance | In-memory lookups | Filer gRPC calls |
## Implementation Considerations
### Advantages of SeaweedFS Approach
- Integrates with existing filer metadata
- Supports both implicit and explicit directories
- Preserves directory metadata and attributes
- Compatible with POSIX filer semantics
### Trade-offs
- Additional filer communication overhead for HEAD requests
- Complexity of supporting both directory paradigms
- Performance depends on filer efficiency
## TODO
- [ ] Add performance benchmark comparison: MinIO vs SeaweedFS
- [ ] Document edge cases where behaviors differ
- [ ] Add example request/response traces for both systems
- [ ] Document migration path for users moving from MinIO to SeaweedFS
- [ ] Add compatibility matrix for different S3 clients
test/s3/parquet/Makefile
+36 -128
View File
@@ -4,14 +4,9 @@
# Default values
SEAWEEDFS_BINARY ?= weed
S3_PORT ?= 8333
FILER_PORT ?= 8888
VOLUME_PORT ?= 8080
MASTER_PORT ?= 9333
TEST_TIMEOUT ?= 15m
ACCESS_KEY ?= some_access_key1
SECRET_KEY ?= some_secret_key1
VOLUME_MAX_SIZE_MB ?= 50
VOLUME_MAX_COUNT ?= 100
BUCKET_NAME ?= test-parquet-bucket
ENABLE_SSE_S3 ?= false
@@ -68,11 +63,7 @@ help:
@echo "Configuration:"
@echo " SEAWEEDFS_BINARY=$(SEAWEEDFS_BINARY)"
@echo " S3_PORT=$(S3_PORT)"
@echo " FILER_PORT=$(FILER_PORT)"
@echo " VOLUME_PORT=$(VOLUME_PORT)"
@echo " MASTER_PORT=$(MASTER_PORT)"
@echo " BUCKET_NAME=$(BUCKET_NAME)"
@echo " VOLUME_MAX_SIZE_MB=$(VOLUME_MAX_SIZE_MB)"
@echo " ENABLE_SSE_S3=$(ENABLE_SSE_S3)"
@echo " PYTHON=$(PYTHON)"
@@ -106,39 +97,25 @@ setup-python: check-python
start-seaweedfs-ci: check-binary
@echo "$(YELLOW)Starting SeaweedFS server for Parquet testing...$(NC)"
# Clean up any existing processes first (CI-safe)
@echo "Cleaning up any existing processes..."
# Clean up any existing processes first (CI-safe) - aggressive cleanup
@echo "Aggressively cleaning up any existing processes on S3 port $(S3_PORT) and master port 9333..."
@if command -v lsof >/dev/null 2>&1; then \
lsof -ti :$(MASTER_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$(VOLUME_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$(FILER_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$$(( $(MASTER_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$$(( $(VOLUME_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$$(( $(FILER_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
for attempt in 1 2 3; do \
lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -9 $$pid 2>/dev/null || true; done; \
lsof -ti :9333 2>/dev/null | head -5 | while read pid; do kill -9 $$pid 2>/dev/null || true; done; \
sleep 1; \
done; \
fi
@sleep 2
@sleep 3
# Create necessary directories
@mkdir -p /tmp/seaweedfs-test-parquet-master
@mkdir -p /tmp/seaweedfs-test-parquet-volume
@mkdir -p /tmp/seaweedfs-test-parquet-filer
@mkdir -p /tmp/seaweedfs-test-parquet
# Clean up any old server logs
@rm -f /tmp/seaweedfs-parquet-*.log || true
# Start master server with volume size limit and explicit gRPC port
@echo "Starting master server..."
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-parquet-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-parquet-master.log 2>&1 &
@sleep 3
# Start volume server with master HTTP port and increased capacity
@echo "Starting volume server..."
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-parquet-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 -preStopSeconds=1 > /tmp/seaweedfs-parquet-volume.log 2>&1 &
@sleep 5
# Start filer server with embedded S3
@echo "Starting filer server with embedded S3..."
# Start weed mini with embedded S3
@echo "Starting weed mini with embedded S3..."
@if [ "$(ENABLE_SSE_S3)" = "true" ]; then \
echo " SSE-S3 encryption: ENABLED"; \
printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"buckets":[{"name":"$(BUCKET_NAME)","encryption":{"sseS3":{"enabled":true}}}]}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" > /tmp/seaweedfs-parquet-s3.json; \
@@ -146,96 +123,43 @@ start-seaweedfs-ci: check-binary
echo " SSE-S3 encryption: DISABLED"; \
printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}]}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" > /tmp/seaweedfs-parquet-s3.json; \
fi
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) nohup $(SEAWEEDFS_BINARY) filer -port=$(FILER_PORT) -port.grpc=$$(( $(FILER_PORT) + 10000 )) -master=127.0.0.1:$(MASTER_PORT) -dataCenter=defaultDataCenter -ip=127.0.0.1 -s3 -s3.port=$(S3_PORT) -s3.config=/tmp/seaweedfs-parquet-s3.json > /tmp/seaweedfs-parquet-filer.log 2>&1 &
@sleep 5
# Wait for S3 service to be ready - use port-based checking for reliability
@echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)"
@for i in $$(seq 1 20); do \
if netstat -an 2>/dev/null | grep -q ":$(S3_PORT).*LISTEN" || \
ss -an 2>/dev/null | grep -q ":$(S3_PORT).*LISTEN" || \
lsof -i :$(S3_PORT) >/dev/null 2>&1; then \
echo "$(GREEN)S3 service is listening on port $(S3_PORT)$(NC)"; \
sleep 1; \
break; \
@$(SEAWEEDFS_BINARY) mini \
-dir=/tmp/seaweedfs-test-parquet \
-ip.bind=0.0.0.0 \
-s3.port=$(S3_PORT) \
-s3.config=/tmp/seaweedfs-parquet-s3.json \
> /tmp/seaweedfs-parquet-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid
@echo "Waiting for S3 service to be fully ready (max 90 seconds)..."
@bash -c 'for i in $$(seq 1 90); do \
if curl -s -H "Authorization: AWS4-HMAC-SHA256 Credential=$(ACCESS_KEY)" http://localhost:$(S3_PORT)/ > /dev/null 2>&1; then \
echo "✅ S3 service is ready"; \
sleep 2; \
exit 0; \
fi; \
if [ $$i -eq 20 ]; then \
echo "$(RED)S3 service failed to start within 20 seconds$(NC)"; \
echo "=== Detailed Logs ==="; \
echo "Master log:"; tail -30 /tmp/seaweedfs-parquet-master.log || true; \
echo "Volume log:"; tail -30 /tmp/seaweedfs-parquet-volume.log || true; \
echo "Filer log:"; tail -30 /tmp/seaweedfs-parquet-filer.log || true; \
echo "=== Port Status ==="; \
netstat -an 2>/dev/null | grep ":$(S3_PORT)" || \
ss -an 2>/dev/null | grep ":$(S3_PORT)" || \
echo "No port listening on $(S3_PORT)"; \
exit 1; \
fi; \
echo "Waiting for S3 service... ($$i/20)"; \
sleep 1; \
done
# Additional wait for filer gRPC to be ready
@echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)"
@sleep 2
# Wait for volume server to register with master and ensure volume assignment works
@echo "$(YELLOW)Waiting for volume assignment to be ready...$(NC)"
@for i in $$(seq 1 30); do \
ASSIGN_RESULT=$$(curl -s "http://localhost:$(MASTER_PORT)/dir/assign?count=1" 2>/dev/null); \
if echo "$$ASSIGN_RESULT" | grep -q '"fid"'; then \
echo "$(GREEN)Volume assignment is ready$(NC)"; \
break; \
fi; \
if [ $$i -eq 30 ]; then \
echo "$(RED)Volume assignment not ready after 30 seconds$(NC)"; \
echo "=== Last assign attempt ==="; \
echo "$$ASSIGN_RESULT"; \
echo "=== Master Status ==="; \
curl -s "http://localhost:$(MASTER_PORT)/dir/status" 2>/dev/null || echo "Failed to get master status"; \
echo "=== Master Logs ==="; \
tail -50 /tmp/seaweedfs-parquet-master.log 2>/dev/null || echo "No master log"; \
echo "=== Volume Logs ==="; \
tail -50 /tmp/seaweedfs-parquet-volume.log 2>/dev/null || echo "No volume log"; \
exit 1; \
fi; \
echo "Waiting for volume assignment... ($$i/30)"; \
sleep 1; \
done
@echo "$(GREEN)SeaweedFS server started successfully for Parquet testing$(NC)"
@echo "Master: http://localhost:$(MASTER_PORT)"
@echo "Volume: http://localhost:$(VOLUME_PORT)"
@echo "Filer: http://localhost:$(FILER_PORT)"
@echo "S3: http://localhost:$(S3_PORT)"
@echo "Volume Max Size: $(VOLUME_MAX_SIZE_MB)MB"
done; \
echo "ERROR S3 service failed to start within 90 seconds"; \
echo "=== Server log output ==="; \
cat /tmp/seaweedfs-parquet-mini.log 2>/dev/null || echo "No startup log available"; \
exit 1'
start-seaweedfs: check-binary
@echo "$(YELLOW)Starting SeaweedFS server for Parquet testing...$(NC)"
@# Use port-based cleanup for consistency and safety
@echo "Cleaning up any existing processes..."
@lsof -ti :$(MASTER_PORT) 2>/dev/null | xargs -r kill -TERM || true
@lsof -ti :$(VOLUME_PORT) 2>/dev/null | xargs -r kill -TERM || true
@lsof -ti :$(FILER_PORT) 2>/dev/null | xargs -r kill -TERM || true
@lsof -ti :$(S3_PORT) 2>/dev/null | xargs -r kill -TERM || true
@# Clean up gRPC ports (HTTP port + 10000)
@lsof -ti :$$(( $(MASTER_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true
@lsof -ti :$$(( $(VOLUME_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true
@lsof -ti :$$(( $(FILER_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true
@sleep 2
@$(MAKE) start-seaweedfs-ci
stop-seaweedfs:
@echo "$(YELLOW)Stopping SeaweedFS server...$(NC)"
@# Use port-based cleanup for consistency and safety
@lsof -ti :$(MASTER_PORT) 2>/dev/null | xargs -r kill -TERM || true
@lsof -ti :$(VOLUME_PORT) 2>/dev/null | xargs -r kill -TERM || true
@lsof -ti :$(FILER_PORT) 2>/dev/null | xargs -r kill -TERM || true
@if [ -f /tmp/weed-mini.pid ]; then \
echo "Stopping weed mini..."; \
kill $$(cat /tmp/weed-mini.pid) || true; \
rm -f /tmp/weed-mini.pid; \
fi
@lsof -ti :$(S3_PORT) 2>/dev/null | xargs -r kill -TERM || true
@# Clean up gRPC ports (HTTP port + 10000)
@lsof -ti :$$(( $(MASTER_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true
@lsof -ti :$$(( $(VOLUME_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true
@lsof -ti :$$(( $(FILER_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true
@sleep 2
@echo "$(GREEN)SeaweedFS server stopped$(NC)"
@@ -245,22 +169,10 @@ stop-seaweedfs-safe:
@# Use port-based cleanup which is safer in CI
@if command -v lsof >/dev/null 2>&1; then \
echo "Using lsof for port-based cleanup..."; \
lsof -ti :$(MASTER_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$(VOLUME_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$(FILER_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$$(( $(MASTER_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$$(( $(VOLUME_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
lsof -ti :$$(( $(FILER_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \
else \
echo "lsof not available, using netstat approach..."; \
netstat -tlnp 2>/dev/null | grep :$(MASTER_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
netstat -tlnp 2>/dev/null | grep :$(VOLUME_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
netstat -tlnp 2>/dev/null | grep :$(FILER_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
netstat -tlnp 2>/dev/null | grep :$(S3_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
netstat -tlnp 2>/dev/null | grep :$$(( $(MASTER_PORT) + 10000 )) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
netstat -tlnp 2>/dev/null | grep :$$(( $(VOLUME_PORT) + 10000 )) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
netstat -tlnp 2>/dev/null | grep :$$(( $(FILER_PORT) + 10000 )) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \
fi
@sleep 2
@echo "$(GREEN)SeaweedFS server safely stopped$(NC)"
@@ -351,18 +263,14 @@ test-implicit-dir-with-server: build-weed setup-python
# Debug targets
debug-logs:
@echo "$(YELLOW)=== Master Log ===$(NC)"
@tail -n 50 /tmp/seaweedfs-parquet-master.log || echo "No master log found"
@echo "$(YELLOW)=== Volume Log ===$(NC)"
@tail -n 50 /tmp/seaweedfs-parquet-volume.log || echo "No volume log found"
@echo "$(YELLOW)=== Filer Log ===$(NC)"
@tail -n 50 /tmp/seaweedfs-parquet-filer.log || echo "No filer log found"
@echo "$(YELLOW)=== Mini Log ===$(NC)"
@tail -n 50 /tmp/seaweedfs-parquet-mini.log || echo "No mini log found"
debug-status:
@echo "$(YELLOW)=== Process Status ===$(NC)"
@ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
@echo "$(YELLOW)=== Port Status ===$(NC)"
@netstat -an | grep -E "($(MASTER_PORT)|$(VOLUME_PORT)|$(FILER_PORT)|$(S3_PORT))" || echo "No ports in use"
@netstat -an | grep -E "($(S3_PORT))" || echo "No ports in use"
# Manual test targets for development
manual-start: start-seaweedfs
test/s3/parquet/TEST_COVERAGE.md
-46
View File
@@ -1,46 +0,0 @@
# Test Coverage Documentation
## Overview
This document provides comprehensive test coverage documentation for the SeaweedFS S3 Parquet integration tests.
## Test Categories
### Unit Tests (Go)
- 17 test cases covering S3 API handlers
- Tests for implicit directory handling
- HEAD request behavior validation
- Located in: `weed/s3api/s3api_implicit_directory_test.go`
### Integration Tests (Python)
- 6 test cases for implicit directory fix
- Tests HEAD request behavior on directory markers
- s3fs directory detection validation
- PyArrow dataset read compatibility
- Located in: `test_implicit_directory_fix.py`
### End-to-End Tests (Python)
- 20 test cases combining write and read methods
- Small file tests (5 rows): 10 test combinations
- Large file tests (200,000 rows): 10 test combinations
- Tests multiple write methods: `pads.write_dataset`, `pq.write_table+s3fs`
- Tests multiple read methods: `pads.dataset`, `pq.ParquetDataset`, `pq.read_table`, `s3fs+direct`, `s3fs+buffered`
- Located in: `s3_parquet_test.py`
## Coverage Summary
| Test Type | Count | Status |
|-----------|-------|--------|
| Unit Tests (Go) | 17 | ✅ Pass |
| Integration Tests (Python) | 6 | ✅ Pass |
| End-to-End Tests (Python) | 20 | ✅ Pass |
| **Total** | **43** | **✅ All Pass** |
## TODO
- [ ] Add detailed test execution time metrics
- [ ] Document test data generation strategies
- [ ] Add code coverage percentages for Go tests
- [ ] Document edge cases and corner cases tested
- [ ] Add performance benchmarking results
test/s3/parquet/debug_write_dataset.py
+86
View File
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""Debug script to understand what pads.write_dataset creates."""
import sys
import pyarrow as pa
import pyarrow.dataset as pads
import s3fs
# Create a simple test table
table = pa.table({'id': [1, 2, 3], 'value': [1.0, 2.0, 3.0]})
# Initialize S3 filesystem
fs = s3fs.S3FileSystem(
client_kwargs={'endpoint_url': 'http://localhost:8333'},
key='some_access_key1',
secret='some_secret_key1',
use_listings_cache=False,
)
# Create bucket
if not fs.exists('test-bucket'):
fs.mkdir('test-bucket')
# Write with pads.write_dataset
test_path = 's3://test-bucket/test-write-simple/'
print(f"Writing to: {test_path}")
print(f"Table schema: {table.schema}")
print(f"Table rows: {table.num_rows}")
try:
pads.write_dataset(table, test_path, format='parquet', filesystem=fs)
print("\n✓ Write succeeded")
# List all files recursively
print(f"\nListing all files recursively under {test_path}:")
import os
base_path = 'test-bucket/test-write-simple'
def list_recursive(path, indent=0):
try:
items = fs.ls(path, detail=False)
for item in items:
is_dir = fs.isdir(item)
item_name = item.split('/')[-1] if '/' in item else item
if is_dir:
print(f"{' ' * indent}📁 {item_name}/")
list_recursive(item, indent + 1)
else:
# Get file size
try:
info = fs.info(item)
size = info.get('size', 0)
print(f"{' ' * indent}📄 {item_name} ({size} bytes)")
except:
print(f"{' ' * indent}📄 {item_name}")
except Exception as e:
print(f"{' ' * indent}Error listing {path}: {e}")
list_recursive(base_path)
# Try to read back with different methods
print(f"\n\nTrying to read back using different methods:")
# Method 1: pads.dataset with the same path
print(f"\n1. pads.dataset('{test_path}'):")
try:
dataset = pads.dataset(test_path, format='parquet', filesystem=fs)
result = dataset.to_table()
print(f" ✓ Success: {result.num_rows} rows")
except Exception as e:
print(f" ✗ Failed: {e}")
# Method 2: pads.dataset with the dir containing parquet files
print(f"\n2. pads.dataset without trailing slash:")
test_path_no_slash = 's3://test-bucket/test-write-simple'
try:
dataset = pads.dataset(test_path_no_slash, format='parquet', filesystem=fs)
result = dataset.to_table()
print(f" ✓ Success: {result.num_rows} rows")
except Exception as e:
print(f" ✗ Failed: {e}")
except Exception as e:
import traceback
print(f"✗ Error: {e}")
traceback.print_exc()
sys.exit(1)
test/s3/parquet/test_implicit_directory_fix.py
+2 -1
View File
@@ -60,6 +60,7 @@ def setup_s3():
endpoint_url=S3_ENDPOINT_URL,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
region_name='us-east-1',
use_ssl=False
)
@@ -182,7 +183,7 @@ def test_explicit_directory_marker(fs, s3_client):
logger.info("="*80)
# Create an explicit directory marker
logger.info(f"\nCreating explicit directory: {BUCKET_NAME}/explicit_dir/")
logger.info(f"Creating explicit directory: {BUCKET_NAME}/explicit_dir/")
try:
s3_client.put_object(
Bucket=BUCKET_NAME,
test/s3/remote_cache/Makefile
+10 -26
View File
@@ -10,19 +10,17 @@ all: test-with-server
# Configuration
WEED_BINARY := ../../../weed/weed_binary
ACCESS_KEY ?= some_access_key1
SECRET_KEY ?= some_secret_key1
# Primary SeaweedFS (the one being tested - has remote caching)
PRIMARY_S3_PORT := 8333
PRIMARY_FILER_PORT := 8888
PRIMARY_MASTER_PORT := 9333
PRIMARY_VOLUME_PORT := 8080
PRIMARY_METRICS_PORT := 9324
PRIMARY_DIR := ./test-primary-data
# Secondary SeaweedFS (acts as "remote" S3 storage)
REMOTE_S3_PORT := 8334
REMOTE_FILER_PORT := 8889
REMOTE_MASTER_PORT := 9334
REMOTE_VOLUME_PORT := 8081
REMOTE_METRICS_PORT := 9325
REMOTE_DIR := ./test-remote-data
@@ -73,18 +71,11 @@ start-remote: check-deps
@echo "Starting remote SeaweedFS (secondary instance)..."
@rm -f remote-server.pid
@mkdir -p $(REMOTE_DIR)
@$(WEED_BINARY) server \
-s3 \
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \
-s3.port=$(REMOTE_S3_PORT) \
-s3.allowDeleteBucketNotEmpty=true \
-filer \
-filer.port=$(REMOTE_FILER_PORT) \
-master.port=$(REMOTE_MASTER_PORT) \
-volume.port=$(REMOTE_VOLUME_PORT) \
-master.volumeSizeLimitMB=50 \
-volume.max=100 \
-dir=$(REMOTE_DIR) \
-volume.preStopSeconds=1 \
-ip.bind=0.0.0.0 \
-metricsPort=$(REMOTE_METRICS_PORT) \
> remote-weed.log 2>&1 & echo $$! > remote-server.pid
@echo "Waiting for remote SeaweedFS to start..."
@@ -93,7 +84,7 @@ start-remote: check-deps
echo "Remote SeaweedFS started on port $(REMOTE_S3_PORT)"; \
exit 0; \
fi; \
sleep 1; \
sleep 3; \
done; \
echo "ERROR: Remote SeaweedFS failed to start"; \
cat remote-weed.log; \
@@ -114,18 +105,11 @@ start-primary: check-deps
@echo "Starting primary SeaweedFS..."
@rm -f primary-server.pid
@mkdir -p $(PRIMARY_DIR)
@$(WEED_BINARY) server \
-s3 \
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \
-s3.port=$(PRIMARY_S3_PORT) \
-s3.allowDeleteBucketNotEmpty=true \
-filer \
-filer.port=$(PRIMARY_FILER_PORT) \
-master.port=$(PRIMARY_MASTER_PORT) \
-volume.port=$(PRIMARY_VOLUME_PORT) \
-master.volumeSizeLimitMB=50 \
-volume.max=100 \
-dir=$(PRIMARY_DIR) \
-volume.preStopSeconds=1 \
-ip.bind=0.0.0.0 \
-metricsPort=$(PRIMARY_METRICS_PORT) \
> primary-weed.log 2>&1 & echo $$! > primary-server.pid
@echo "Waiting for primary SeaweedFS to start..."
@@ -134,7 +118,7 @@ start-primary: check-deps
echo "Primary SeaweedFS started on port $(PRIMARY_S3_PORT)"; \
exit 0; \
fi; \
sleep 1; \
sleep 3; \
done; \
echo "ERROR: Primary SeaweedFS failed to start"; \
cat primary-weed.log; \
@@ -156,7 +140,7 @@ setup-remote:
@curl -s -X PUT "http://localhost:$(REMOTE_S3_PORT)/$(REMOTE_BUCKET)" || echo "Bucket may already exist"
@sleep 1
@echo "Configuring remote storage on primary..."
@printf 'remote.configure -name=seaweedremote -type=s3 -s3.access_key=any -s3.secret_key=any -s3.endpoint=http://localhost:$(REMOTE_S3_PORT) -s3.region=us-east-1\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.configure done"
@printf 'remote.configure -name=seaweedremote -type=s3 -s3.access_key=$(ACCESS_KEY) -s3.secret_key=$(SECRET_KEY) -s3.endpoint=http://localhost:$(REMOTE_S3_PORT) -s3.region=us-east-1\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.configure done"
@sleep 2
@echo "Mounting remote bucket on primary..."
@printf 'remote.mount -dir=/buckets/remotemounted -remote=seaweedremote/$(REMOTE_BUCKET) -nonempty\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.mount done"
test/s3/remote_cache/remote_cache_test.go
+2 -13
View File
@@ -34,8 +34,8 @@ const (
remoteEndpoint = "http://localhost:8334"
// Credentials (anonymous access for testing)
accessKey = "any"
secretKey = "any"
accessKey = "some_access_key1"
secretKey = "some_secret_key1"
// Bucket name - mounted on primary as remote storage
testBucket = "remotemounted"
@@ -121,17 +121,6 @@ func getFromPrimary(t *testing.T, key string) []byte {
return data
}
// syncToRemote syncs local data to remote storage
func syncToRemote(t *testing.T) {
t.Log("Syncing to remote storage...")
output, err := runWeedShell(t, "remote.cache.uncache -dir=/buckets/"+testBucket+" -include=*")
if err != nil {
t.Logf("syncToRemote warning: %v", err)
}
t.Log(output)
time.Sleep(1 * time.Second)
}
// uncacheLocal purges the local cache, forcing data to be fetched from remote
func uncacheLocal(t *testing.T, pattern string) {
t.Logf("Purging local cache for pattern: %s", pattern)
test/s3/retention/Makefile
+12 -19
View File
@@ -6,11 +6,14 @@
# Configuration
WEED_BINARY := ../../../weed/weed_binary
S3_PORT := 8333
ACCESS_KEY ?= some_access_key1
SECRET_KEY ?= some_secret_key1
MASTER_PORT := 9333
VOLUME_PORT := 8080
FILER_PORT := 8888
TEST_TIMEOUT := 15m
TEST_PATTERN := TestRetention
SERVER_DIR := ./test-volume-data/server-data
# Default target
help:
@@ -80,23 +83,16 @@ start-server: check-deps
@ls -la ../../../docker/compose/s3.json || echo "⚠️ Config file not found, continuing without it"
@echo "🔍 DEBUG: Creating volume directory..."
@mkdir -p ./test-volume-data
@echo "🔍 DEBUG: Launching SeaweedFS server in background..."
@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
@$(WEED_BINARY) server \
-debug \
-s3 \
@echo "🔍 DEBUG: Creating server data directory..."
@mkdir -p $(SERVER_DIR)
@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT)"
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \
-dir=$(SERVER_DIR) \
-s3.port=$(S3_PORT) \
-s3.allowDeleteBucketNotEmpty=true \
-s3.config=../../../docker/compose/s3.json \
-filer \
-filer.maxMB=64 \
-master.volumeSizeLimitMB=50 \
-volume.max=100 \
-dir=./test-volume-data \
-volume.preStopSeconds=1 \
-metricsPort=9324 \
> weed-test.log 2>&1 & echo $$! > weed-server.pid
@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
> weed-test.log 2>&1 & \
echo $$! > weed-server.pid
@echo "🔍 DEBUG: Server PID: $$(cat weed-test.pid 2>/dev/null || echo 'PID file not found')"
@echo "🔍 DEBUG: Checking if PID is still running..."
@sleep 2
@if [ -f weed-server.pid ]; then \
@@ -105,7 +101,6 @@ start-server: check-deps
else \
echo "⚠️ PID file not found"; \
fi
@echo "🔍 DEBUG: Waiting for server to start (up to 90 seconds)..."
@for i in $$(seq 1 90); do \
echo "🔍 DEBUG: Attempt $$i/90 - checking port $(S3_PORT)"; \
if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \
@@ -123,8 +118,6 @@ start-server: check-deps
if [ $$i -eq 15 ]; then \
echo "🔍 DEBUG: After 15 seconds, checking port bindings..."; \
netstat -tlnp 2>/dev/null | grep $(S3_PORT) || echo "Port $(S3_PORT) not bound"; \
netstat -tlnp 2>/dev/null | grep 9333 || echo "Port 9333 not bound"; \
netstat -tlnp 2>/dev/null | grep 8080 || echo "Port 8080 not bound"; \
fi; \
if [ $$i -eq 30 ]; then \
echo "⚠️ Server taking longer than expected (30s), checking logs..."; \
test/s3/sse/Makefile
+62 -86
View File
@@ -93,54 +93,35 @@ start-seaweedfs: check-binary
@sleep 2
# Create necessary directories
@mkdir -p /tmp/seaweedfs-test-sse-master
@mkdir -p /tmp/seaweedfs-test-sse-volume
@mkdir -p /tmp/seaweedfs-test-sse-filer
# Start master server with volume size limit and explicit gRPC port
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-sse-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-sse-master.log 2>&1 &
@sleep 3
# Start volume server with master HTTP port and increased capacity
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-sse-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 > /tmp/seaweedfs-sse-volume.log 2>&1 &
@sleep 5
# Start filer server (using standard SeaweedFS gRPC port convention: HTTP port + 10000)
@nohup $(SEAWEEDFS_BINARY) filer -port=$(FILER_PORT) -port.grpc=$$(( $(FILER_PORT) + 10000 )) -master=127.0.0.1:$(MASTER_PORT) -dataCenter=defaultDataCenter -ip=127.0.0.1 > /tmp/seaweedfs-sse-filer.log 2>&1 &
@sleep 3
@mkdir -p /tmp/seaweedfs-test-sse
# Create S3 configuration with SSE-KMS support
@printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"kms":{"type":"%s","configs":{"keyId":"%s","encryptionContext":{},"bucketKey":false}}}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" "$(KMS_TYPE)" "$(KMS_KEY_ID)" > /tmp/seaweedfs-sse-s3.json
# Start S3 server with KMS configuration
@nohup $(SEAWEEDFS_BINARY) s3 -port=$(S3_PORT) -filer=127.0.0.1:$(FILER_PORT) -config=/tmp/seaweedfs-sse-s3.json -ip.bind=127.0.0.1 > /tmp/seaweedfs-sse-s3.log 2>&1 &
@sleep 5
# Wait for S3 service to be ready
@echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)"
# Start weed mini
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(SEAWEEDFS_BINARY) mini \
-dir=/tmp/seaweedfs-test-sse \
-s3.port=$(S3_PORT) \
-s3.config=/tmp/seaweedfs-sse-s3.json \
> /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid
@echo "Checking S3 service is ready..."
@for i in $$(seq 1 30); do \
if curl -s -f http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
echo "$(GREEN)S3 service is ready$(NC)"; \
if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
echo "✅ S3 service is ready"; \
break; \
fi; \
echo "Waiting for S3 service... ($$i/30)"; \
sleep 1; \
done
# Additional wait for filer gRPC to be ready
@echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)"
@sleep 2
@echo "$(GREEN)SeaweedFS server started successfully for SSE testing$(NC)"
@echo "Master: http://localhost:$(MASTER_PORT)"
@echo "Volume: http://localhost:$(VOLUME_PORT)"
@echo "Filer: http://localhost:$(FILER_PORT)"
@echo "S3: http://localhost:$(S3_PORT)"
@echo "Volume Max Size: $(VOLUME_MAX_SIZE_MB)MB"
@echo "SSE-KMS Support: Enabled"
stop-seaweedfs:
@echo "$(YELLOW)Stopping SeaweedFS server...$(NC)"
@# Use port-based cleanup for consistency and safety
@if [ -f /tmp/weed-mini.pid ]; then \
echo "Stopping weed mini..."; \
kill $$(cat /tmp/weed-mini.pid) || true; \
rm -f /tmp/weed-mini.pid; \
fi
@lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM || true
@lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM || true
@lsof -ti :$(FILER_PORT) | xargs -r kill -TERM || true
@@ -345,71 +326,33 @@ start-seaweedfs-ci: check-binary
@echo "$(YELLOW)Starting SeaweedFS server for CI testing...$(NC)"
# Create necessary directories
@mkdir -p /tmp/seaweedfs-test-sse-master
@mkdir -p /tmp/seaweedfs-test-sse-volume
@mkdir -p /tmp/seaweedfs-test-sse-filer
@mkdir -p /tmp/seaweedfs-test-sse
# Clean up any old server logs
@rm -f /tmp/seaweedfs-sse-*.log || true
# Start master server with volume size limit and explicit gRPC port
@echo "Starting master server..."
@nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-sse-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-sse-master.log 2>&1 &
@sleep 3
# Start volume server with master HTTP port and increased capacity
@echo "Starting volume server..."
@nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-sse-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 > /tmp/seaweedfs-sse-volume.log 2>&1 &
@sleep 5
# Create S3 JSON configuration with KMS (Local provider) and basic identity for embedded S3
@sed -e 's/ACCESS_KEY_PLACEHOLDER/$(ACCESS_KEY)/g' \
-e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
s3-config-template.json > /tmp/seaweedfs-s3.json
# Start filer server with embedded S3 using the JSON config (with verbose logging)
@echo "Starting filer server with embedded S3..."
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 nohup $(SEAWEEDFS_BINARY) filer -port=$(FILER_PORT) -port.grpc=$$(( $(FILER_PORT) + 10000 )) -master=127.0.0.1:$(MASTER_PORT) -dataCenter=defaultDataCenter -ip=127.0.0.1 -s3 -s3.port=$(S3_PORT) -s3.config=/tmp/seaweedfs-s3.json > /tmp/seaweedfs-sse-filer.log 2>&1 &
@sleep 5
# Start weed mini with embedded S3 using the JSON config (with verbose logging)
@echo "Starting weed mini with embedded S3..."
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
-dir=/tmp/seaweedfs-test-sse \
-s3.port=$(S3_PORT) \
-s3.config=/tmp/seaweedfs-s3.json \
-ip=127.0.0.1 \
> /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid
# Wait for S3 service to be ready - use port-based checking for reliability
@echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)"
@for i in $$(seq 1 20); do \
if netstat -an 2>/dev/null | grep -q ":$(S3_PORT).*LISTEN" || \
ss -an 2>/dev/null | grep -q ":$(S3_PORT).*LISTEN" || \
lsof -i :$(S3_PORT) >/dev/null 2>&1; then \
echo "$(GREEN)S3 service is listening on port $(S3_PORT)$(NC)"; \
sleep 1; \
@echo "Checking S3 service is ready..."
@for i in $$(seq 1 30); do \
if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
echo "✅ S3 service is ready"; \
break; \
fi; \
if [ $$i -eq 20 ]; then \
echo "$(RED)S3 service failed to start within 20 seconds$(NC)"; \
echo "=== Detailed Logs ==="; \
echo "Master log:"; tail -30 /tmp/seaweedfs-sse-master.log || true; \
echo "Volume log:"; tail -30 /tmp/seaweedfs-sse-volume.log || true; \
echo "Filer log:"; tail -30 /tmp/seaweedfs-sse-filer.log || true; \
echo "=== Port Status ==="; \
netstat -an 2>/dev/null | grep ":$(S3_PORT)" || \
ss -an 2>/dev/null | grep ":$(S3_PORT)" || \
echo "No port listening on $(S3_PORT)"; \
echo "=== Process Status ==="; \
ps aux | grep -E "weed.*(filer|s3).*$(S3_PORT)" | grep -v grep || echo "No S3 process found"; \
exit 1; \
fi; \
echo "Waiting for S3 service... ($$i/20)"; \
sleep 1; \
done
# Additional wait for filer gRPC to be ready
@echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)"
@sleep 2
@echo "$(GREEN)SeaweedFS server started successfully for SSE testing$(NC)"
@echo "Master: http://localhost:$(MASTER_PORT)"
@echo "Volume: http://localhost:$(VOLUME_PORT)"
@echo "Filer: http://localhost:$(FILER_PORT)"
@echo "S3: http://localhost:$(S3_PORT)"
@echo "Volume Max Size: $(VOLUME_MAX_SIZE_MB)MB"
@echo "SSE-KMS Support: Enabled"
# GitHub Actions compatible quick test subset
test-quick-with-server: build-weed
@@ -527,3 +470,36 @@ dev-kms: setup-openbao
@echo "OpenBao: $(OPENBAO_ADDR)"
@echo "Token: $(OPENBAO_TOKEN)"
@echo "Use 'make test-ssekms-integration' to run tests"
# Volume encryption integration tests
test-volume-encryption: build-weed
@echo "🚀 Starting S3 volume encryption integration tests..."
@echo "Starting SeaweedFS cluster with volume encryption enabled..."
@# Start server with -s3.encryptVolumeData flag
@mkdir -p /tmp/seaweedfs-test-sse
@rm -f /tmp/seaweedfs-sse-*.log || true
@sed -e 's/ACCESS_KEY_PLACEHOLDER/$(ACCESS_KEY)/g' \
-e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \
s3-config-template.json > /tmp/seaweedfs-s3.json
@echo "Starting weed mini with S3 volume encryption..."
@AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \
-dir=/tmp/seaweedfs-test-sse \
-s3.port=$(S3_PORT) \
-s3.config=/tmp/seaweedfs-s3.json \
-s3.encryptVolumeData \
-ip=127.0.0.1 \
> /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid
@echo "Checking S3 service is ready..."
@for i in $$(seq 1 30); do \
if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \
echo "✅ S3 service is ready"; \
break; \
fi; \
sleep 1; \
done
@echo "Running volume encryption integration tests..."
@trap '$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true' EXIT; \
cd $(SEAWEEDFS_ROOT) && go test -v -tags=integration -timeout=10m -run "TestS3VolumeEncryption" ./test/s3/sse || exit 1; \
echo "✅ Volume encryption tests completed successfully"; \
$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true
test/s3/sse/s3_volume_encryption_test.go
+448
View File
@@ -0,0 +1,448 @@
//go:build integration
// +build integration
package sse
import (
"bytes"
"context"
"fmt"
"io"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
// TestS3VolumeEncryptionRoundtrip tests that data uploaded with encryptVolumeData
// enabled can be read back correctly. This requires the S3 server to be started with
// -encryptVolumeData=true for the test to verify encryption is working.
//
// To run this test:
// 1. Start SeaweedFS: weed server -s3 -s3.encryptVolumeData=true
// 2. Run: go test -v -run TestS3VolumeEncryptionRoundtrip
func TestS3VolumeEncryptionRoundtrip(t *testing.T) {
svc := getS3Client(t)
bucket := fmt.Sprintf("volume-encryption-test-%d", time.Now().Unix())
// Create bucket
_, err := svc.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
t.Fatalf("Failed to create bucket: %v", err)
}
defer cleanupBucket(t, svc, bucket)
testCases := []struct {
name string
key string
content string
rangeReq string // Optional range request
}{
{
name: "small file",
key: "small.txt",
content: "Hello, encrypted world!",
},
{
name: "medium file",
key: "medium.txt",
content: strings.Repeat("SeaweedFS volume encryption test content. ", 1000),
},
{
name: "binary content",
key: "binary.bin",
content: string([]byte{0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x00, 0x80}),
},
{
name: "range request",
key: "range-test.txt",
content: "0123456789ABCDEFGHIJ",
rangeReq: "bytes=5-10",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
// Upload
_, err := svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(tc.key),
Body: strings.NewReader(tc.content),
})
if err != nil {
t.Fatalf("PutObject failed: %v", err)
}
// Download
getInput := &s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(tc.key),
}
if tc.rangeReq != "" {
getInput.Range = aws.String(tc.rangeReq)
}
result, err := svc.GetObject(getInput)
if err != nil {
t.Fatalf("GetObject failed: %v", err)
}
defer result.Body.Close()
data, err := io.ReadAll(result.Body)
if err != nil {
t.Fatalf("Failed to read response body: %v", err)
}
// Verify content
expected := tc.content
if tc.rangeReq != "" {
// For "bytes=5-10", we expect characters at positions 5-10 (inclusive)
expected = tc.content[5:11]
}
if string(data) != expected {
t.Errorf("Content mismatch:\n expected: %q\n got: %q", expected, string(data))
} else {
t.Logf("Successfully uploaded and downloaded %s (%d bytes)", tc.key, len(data))
}
})
}
}
// TestS3VolumeEncryptionMultiChunk tests large files that span multiple chunks
// to ensure encryption works correctly across chunk boundaries.
func TestS3VolumeEncryptionMultiChunk(t *testing.T) {
svc := getS3Client(t)
bucket := fmt.Sprintf("volume-encryption-multichunk-%d", time.Now().Unix())
// Create bucket
_, err := svc.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
t.Fatalf("Failed to create bucket: %v", err)
}
defer cleanupBucket(t, svc, bucket)
// Create a file larger than default chunk size (8MB)
// Use 10MB to ensure multiple chunks
largeContent := make([]byte, 10*1024*1024)
for i := range largeContent {
largeContent[i] = byte(i % 256)
}
key := "large-file.bin"
// Upload
_, err = svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: bytes.NewReader(largeContent),
})
if err != nil {
t.Fatalf("PutObject failed for large file: %v", err)
}
// Download full file
result, err := svc.GetObject(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
if err != nil {
t.Fatalf("GetObject failed: %v", err)
}
defer result.Body.Close()
downloadedData, err := io.ReadAll(result.Body)
if err != nil {
t.Fatalf("Failed to read response body: %v", err)
}
if len(downloadedData) != len(largeContent) {
t.Errorf("Size mismatch: expected %d, got %d", len(largeContent), len(downloadedData))
}
if !bytes.Equal(downloadedData, largeContent) {
t.Errorf("Content mismatch in multi-chunk file")
// Find first mismatch
for i := 0; i < len(downloadedData) && i < len(largeContent); i++ {
if downloadedData[i] != largeContent[i] {
t.Errorf("First mismatch at byte %d: expected %02x, got %02x", i, largeContent[i], downloadedData[i])
break
}
}
} else {
t.Logf("Successfully uploaded and downloaded %d byte multi-chunk file", len(downloadedData))
}
// Test range request spanning chunk boundary (around 8MB)
rangeStart := int64(8*1024*1024 - 1000)
rangeEnd := int64(8*1024*1024 + 1000)
rangeResult, err := svc.GetObject(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Range: aws.String(fmt.Sprintf("bytes=%d-%d", rangeStart, rangeEnd)),
})
if err != nil {
t.Fatalf("Range GetObject failed: %v", err)
}
defer rangeResult.Body.Close()
rangeData, err := io.ReadAll(rangeResult.Body)
if err != nil {
t.Fatalf("Failed to read range response: %v", err)
}
expectedRange := largeContent[rangeStart : rangeEnd+1]
if !bytes.Equal(rangeData, expectedRange) {
t.Errorf("Range request content mismatch at chunk boundary")
} else {
t.Logf("Successfully retrieved range spanning chunk boundary (%d bytes)", len(rangeData))
}
}
// TestS3VolumeEncryptionMultiChunkRangeRead tests range reads that span multiple chunks:
// - Part of chunk 1
// - Whole chunk 2
// - Part of chunk 3
// This is critical for verifying that cipher decryption works correctly when
// reading across chunk boundaries with the cipherKey from each chunk.
func TestS3VolumeEncryptionMultiChunkRangeRead(t *testing.T) {
svc := getS3Client(t)
bucket := fmt.Sprintf("volume-encryption-multirange-%d", time.Now().Unix())
// Create bucket
_, err := svc.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
t.Fatalf("Failed to create bucket: %v", err)
}
defer cleanupBucket(t, svc, bucket)
// Default chunk size is 8MB. Create a file with 3+ chunks (25MB)
// to ensure we have multiple complete chunks
const chunkSize = 8 * 1024 * 1024 // 8MB
const fileSize = 25 * 1024 * 1024 // 25MB = 3 chunks + partial 4th
largeContent := make([]byte, fileSize)
for i := range largeContent {
// Use recognizable pattern: each byte encodes its position
largeContent[i] = byte(i % 256)
}
key := "multi-chunk-range-test.bin"
// Upload
_, err = svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: bytes.NewReader(largeContent),
})
if err != nil {
t.Fatalf("PutObject failed: %v", err)
}
t.Logf("Uploaded %d byte file (%d chunks of %d bytes)", fileSize, (fileSize+chunkSize-1)/chunkSize, chunkSize)
// Test cases for range reads spanning multiple chunks
testCases := []struct {
name string
rangeStart int64
rangeEnd int64
desc string
}{
{
name: "part_chunk1_whole_chunk2_part_chunk3",
rangeStart: chunkSize - 1000, // 1000 bytes before end of chunk 1
rangeEnd: 2*chunkSize + 1000, // 1000 bytes into chunk 3
desc: "Spans from end of chunk 1 through entire chunk 2 into beginning of chunk 3",
},
{
name: "last_byte_chunk1_through_first_byte_chunk3",
rangeStart: chunkSize - 1, // Last byte of chunk 1
rangeEnd: 2 * chunkSize, // First byte of chunk 3
desc: "Minimal span: last byte of chunk 1, entire chunk 2, first byte of chunk 3",
},
{
name: "middle_chunk1_to_middle_chunk3",
rangeStart: chunkSize / 2, // Middle of chunk 1
rangeEnd: 2*chunkSize + chunkSize/2, // Middle of chunk 3
desc: "From middle of chunk 1 to middle of chunk 3",
},
{
name: "half_chunk1_whole_chunk2_half_chunk3",
rangeStart: chunkSize / 2, // Half of chunk 1
rangeEnd: 2*chunkSize + chunkSize/2 - 1, // Half of chunk 3
desc: "Half of each boundary chunk, whole middle chunk",
},
{
name: "single_byte_each_boundary",
rangeStart: chunkSize - 1, // 1 byte from chunk 1
rangeEnd: 2 * chunkSize, // 1 byte from chunk 3
desc: "1 byte from chunk 1, entire chunk 2, 1 byte from chunk 3",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
t.Logf("Test: %s", tc.desc)
t.Logf("Range: bytes=%d-%d (spanning bytes at offsets across chunk boundaries)", tc.rangeStart, tc.rangeEnd)
result, err := svc.GetObject(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Range: aws.String(fmt.Sprintf("bytes=%d-%d", tc.rangeStart, tc.rangeEnd)),
})
if err != nil {
t.Fatalf("Range GetObject failed: %v", err)
}
defer result.Body.Close()
rangeData, err := io.ReadAll(result.Body)
if err != nil {
t.Fatalf("Failed to read range response: %v", err)
}
expectedLen := tc.rangeEnd - tc.rangeStart + 1
if int64(len(rangeData)) != expectedLen {
t.Errorf("Size mismatch: expected %d bytes, got %d", expectedLen, len(rangeData))
}
expectedRange := largeContent[tc.rangeStart : tc.rangeEnd+1]
if !bytes.Equal(rangeData, expectedRange) {
t.Errorf("Content mismatch in multi-chunk range read")
// Find first mismatch for debugging
for i := 0; i < len(rangeData) && i < len(expectedRange); i++ {
if rangeData[i] != expectedRange[i] {
globalOffset := tc.rangeStart + int64(i)
chunkNum := globalOffset / chunkSize
offsetInChunk := globalOffset % chunkSize
t.Errorf("First mismatch at byte %d (chunk %d, offset %d in chunk): expected %02x, got %02x",
i, chunkNum, offsetInChunk, expectedRange[i], rangeData[i])
break
}
}
} else {
t.Logf("✓ Successfully read %d bytes spanning chunks (offsets %d-%d)", len(rangeData), tc.rangeStart, tc.rangeEnd)
}
})
}
}
// TestS3VolumeEncryptionCopy tests that copying encrypted objects works correctly.
func TestS3VolumeEncryptionCopy(t *testing.T) {
svc := getS3Client(t)
bucket := fmt.Sprintf("volume-encryption-copy-%d", time.Now().Unix())
// Create bucket
_, err := svc.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
t.Fatalf("Failed to create bucket: %v", err)
}
defer cleanupBucket(t, svc, bucket)
srcKey := "source-object.txt"
dstKey := "copied-object.txt"
content := "Content to be copied with volume-level encryption"
// Upload source
_, err = svc.PutObject(&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(srcKey),
Body: strings.NewReader(content),
})
if err != nil {
t.Fatalf("PutObject failed: %v", err)
}
// Copy object
_, err = svc.CopyObject(&s3.CopyObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(dstKey),
CopySource: aws.String(fmt.Sprintf("%s/%s", bucket, srcKey)),
})
if err != nil {
t.Fatalf("CopyObject failed: %v", err)
}
// Read copied object
result, err := svc.GetObject(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(dstKey),
})
if err != nil {
t.Fatalf("GetObject failed for copied object: %v", err)
}
defer result.Body.Close()
data, err := io.ReadAll(result.Body)
if err != nil {
t.Fatalf("Failed to read copied object: %v", err)
}
if string(data) != content {
t.Errorf("Copied content mismatch:\n expected: %q\n got: %q", content, string(data))
} else {
t.Logf("Successfully copied encrypted object")
}
}
// Helper functions
func getS3Client(t *testing.T) *s3.S3 {
// Use credentials that match the Makefile configuration
// ACCESS_KEY ?= some_access_key1
// SECRET_KEY ?= some_secret_key1
sess, err := session.NewSession(&aws.Config{
Region: aws.String("us-east-1"),
Endpoint: aws.String("http://localhost:8333"),
DisableSSL: aws.Bool(true),
S3ForcePathStyle: aws.Bool(true),
Credentials: credentials.NewStaticCredentials("some_access_key1", "some_secret_key1", ""),
})
if err != nil {
t.Fatalf("Failed to create session: %v", err)
}
return s3.New(sess)
}
func cleanupBucket(t *testing.T, svc *s3.S3, bucket string) {
ctx := context.Background()
_ = ctx
// List and delete all objects
listResult, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
Bucket: aws.String(bucket),
})
if err != nil {
t.Logf("Warning: failed to list objects for cleanup: %v", err)
return
}
for _, obj := range listResult.Contents {
_, err := svc.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: obj.Key,
})
if err != nil {
t.Logf("Warning: failed to delete object %s: %v", *obj.Key, err)
}
}
// Delete bucket
_, err = svc.DeleteBucket(&s3.DeleteBucketInput{
Bucket: aws.String(bucket),
})
if err != nil {
t.Logf("Warning: failed to delete bucket %s: %v", bucket, err)
}
}
test/s3/tagging/Makefile
+5 -31
View File
@@ -72,41 +72,17 @@ start-server: check-deps
fi
@echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)"
@ls -la $(WEED_BINARY) || (echo "❌ Binary not found!" && exit 1)
@echo "🔍 DEBUG: Checking config file at ../../../docker/compose/s3.json"
@ls -la ../../../docker/compose/s3.json || echo "⚠️ Config file not found, continuing without it"
@echo "🔍 DEBUG: Creating volume directory..."
@mkdir -p ./test-volume-data
@echo "🔍 DEBUG: Launching SeaweedFS server in background..."
@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 -dir=./test-volume-data -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 -volume.max=100 -volume.preStopSeconds=1 -master.port=$(MASTER_PORT) -volume.port=$(VOLUME_PORT) -filer.port=$(FILER_PORT) -s3.port=$(S3_PORT) -metricsPort=9329 -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -master.peers=none"
@$(WEED_BINARY) server \
-filer \
@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=./test-volume-data -s3.port=$(S3_PORT)"
@$(WEED_BINARY) mini \
-filer.maxMB=64 \
-s3 \
-ip.bind 0.0.0.0 \
-dir=./test-volume-data \
-master.raftHashicorp \
-master.electionTimeout 1s \
-master.volumeSizeLimitMB=100 \
-volume.max=100 \
-volume.preStopSeconds=1 \
-master.port=$(MASTER_PORT) \
-volume.port=$(VOLUME_PORT) \
-filer.port=$(FILER_PORT) \
-s3.port=$(S3_PORT) \
-metricsPort=9329 \
-s3.allowDeleteBucketNotEmpty=true \
-s3.config=../../../docker/compose/s3.json \
-master.peers=none \
> weed-test.log 2>&1 & echo $$! > weed-server.pid
@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
@echo "🔍 DEBUG: Checking if PID is still running..."
@sleep 2
@if [ -f weed-server.pid ]; then \
SERVER_PID=$$(cat weed-server.pid); \
ps -p $$SERVER_PID || echo "⚠️ Server PID $$SERVER_PID not found after 2 seconds"; \
else \
echo "⚠️ PID file not found"; \
fi
> weed-test.log 2>&1 & \
echo $$! > weed-server.pid
@echo "🔍 DEBUG: Waiting for server to start (up to 90 seconds)..."
@for i in $$(seq 1 90); do \
echo "🔍 DEBUG: Attempt $$i/90 - checking port $(S3_PORT)"; \
@@ -125,8 +101,6 @@ start-server: check-deps
if [ $$i -eq 15 ]; then \
echo "🔍 DEBUG: After 15 seconds, checking port bindings..."; \
netstat -tlnp 2>/dev/null | grep $(S3_PORT) || echo "Port $(S3_PORT) not bound"; \
netstat -tlnp 2>/dev/null | grep $(MASTER_PORT) || echo "Port $(MASTER_PORT) not bound"; \
netstat -tlnp 2>/dev/null | grep $(VOLUME_PORT) || echo "Port $(VOLUME_PORT) not bound"; \
fi; \
if [ $$i -eq 30 ]; then \
echo "⚠️ Server taking longer than expected (30s), checking logs..."; \
@@ -141,7 +115,7 @@ start-server: check-deps
echo "🔍 DEBUG: Final process check:"; \
ps aux | grep weed | grep -v grep || echo "No weed processes found"; \
echo "🔍 DEBUG: Final port check:"; \
netstat -tlnp 2>/dev/null | grep -E "($(S3_PORT)|$(MASTER_PORT)|$(VOLUME_PORT))" || echo "No ports bound"; \
netstat -tlnp 2>/dev/null | grep -E "($(S3_PORT))" || echo "No ports bound"; \
echo "=== Full server logs ==="; \
if [ -f weed-test.log ]; then \
cat weed-test.log; \
test/s3/tagging/s3_tagging_test.go
+28 -7
View File
@@ -63,18 +63,12 @@ func getS3Client(t *testing.T) *s3.Client {
defaultConfig.SecretKey,
"",
)),
config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
func(service, region string, options ...interface{}) (aws.Endpoint, error) {
return aws.Endpoint{
URL: defaultConfig.Endpoint,
SigningRegion: defaultConfig.Region,
}, nil
})),
)
require.NoError(t, err)
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = true
o.BaseEndpoint = aws.String(defaultConfig.Endpoint)
})
return client
}
@@ -113,6 +107,33 @@ func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) {
}
}
// Delete all versions and delete markers if versioning is enabled
listVersionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
Bucket: aws.String(bucketName),
})
if err == nil {
for _, version := range listVersionsResp.Versions {
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: version.Key,
VersionId: version.VersionId,
})
if err != nil {
t.Logf("Warning: failed to delete version %s: %v", *version.Key, err)
}
}
for _, marker := range listVersionsResp.DeleteMarkers {
_, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: marker.Key,
VersionId: marker.VersionId,
})
if err != nil {
t.Logf("Warning: failed to delete marker %s: %v", *marker.Key, err)
}
}
}
// Then delete the bucket
_, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{
Bucket: aws.String(bucketName),
test/s3/tagging/s3_tagging_versioning_test.go
+434
View File
@@ -0,0 +1,434 @@
package tagging
import (
"context"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// This is the fix for GitHub issue #7868 where tagging failed with "no entry is found in filer store"
// TestPutObjectTaggingOnVersionedBucket tests setting tags on objects in a versioned bucket
func TestPutObjectTaggingOnVersionedBucket(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
// Put object in versioned bucket
objectKey := "versioned-object-with-tags"
objectContent := "Hello, Versioned World!"
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Body: strings.NewReader(objectContent),
Key: aws.String(objectKey),
Bucket: aws.String(bucketName),
})
require.NoError(t, err, "Should be able to put object in versioned bucket")
// Set tags on the object in versioned bucket
_, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Tagging: &types.Tagging{
TagSet: []types.Tag{
{
Key: aws.String("env"),
Value: aws.String("production"),
},
{
Key: aws.String("team"),
Value: aws.String("platform"),
},
},
},
})
require.NoError(t, err, "Should be able to put tags on versioned object")
// Get the tags back
tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err, "Should be able to get tags from versioned object")
// Verify tags
assert.Len(t, tagResp.TagSet, 2, "Should have 2 tags")
tagMap := make(map[string]string)
for _, tag := range tagResp.TagSet {
tagMap[*tag.Key] = *tag.Value
}
assert.Equal(t, "production", tagMap["env"], "env tag should be 'production'")
assert.Equal(t, "platform", tagMap["team"], "team tag should be 'platform'")
}
// TestPutObjectTaggingOnSpecificVersionInVersionedBucket tests setting tags on a specific version
func TestPutObjectTaggingOnSpecificVersionInVersionedBucket(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
// Create multiple versions of the object
objectKey := "multi-version-object"
version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Body: strings.NewReader("Version 1"),
Key: aws.String(objectKey),
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
// Small delay to ensure different version IDs
time.Sleep(50 * time.Millisecond)
versionId1 := *version1Resp.VersionId
version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Body: strings.NewReader("Version 2"),
Key: aws.String(objectKey),
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
versionId2 := *version2Resp.VersionId
// Set tags on version 1
_, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
Tagging: &types.Tagging{
TagSet: []types.Tag{
{
Key: aws.String("version"),
Value: aws.String("v1"),
},
},
},
})
require.NoError(t, err, "Should be able to put tags on specific version 1")
// Set tags on version 2
_, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
Tagging: &types.Tagging{
TagSet: []types.Tag{
{
Key: aws.String("version"),
Value: aws.String("v2"),
},
},
},
})
require.NoError(t, err, "Should be able to put tags on specific version 2")
// Get tags from version 1
tagResp1, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
})
require.NoError(t, err, "Should be able to get tags from version 1")
assert.Len(t, tagResp1.TagSet, 1, "Version 1 should have 1 tag")
assert.Equal(t, "v1", *tagResp1.TagSet[0].Value, "Version 1 tag value should be 'v1'")
// Get tags from version 2
tagResp2, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
})
require.NoError(t, err, "Should be able to get tags from version 2")
assert.Len(t, tagResp2.TagSet, 1, "Version 2 should have 1 tag")
assert.Equal(t, "v2", *tagResp2.TagSet[0].Value, "Version 2 tag value should be 'v2'")
}
// TestDeleteObjectTaggingOnVersionedBucket tests deleting tags from versioned objects
func TestDeleteObjectTaggingOnVersionedBucket(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
// Put object with tags in versioned bucket
objectKey := "versioned-object-tag-delete"
objectContent := "Hello, Delete Tags!"
_, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Body: strings.NewReader(objectContent),
Key: aws.String(objectKey),
Bucket: aws.String(bucketName),
Tagging: aws.String("env=dev&purpose=testing"),
})
require.NoError(t, err)
// Verify tags exist
tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err)
assert.Len(t, tagResp.TagSet, 2, "Should have 2 tags before deletion")
// Delete tags from the versioned object
_, err = client.DeleteObjectTagging(context.TODO(), &s3.DeleteObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err, "Should be able to delete tags from versioned object")
// Verify tags are deleted
tagResp, err = client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err)
assert.Len(t, tagResp.TagSet, 0, "Should have 0 tags after deletion")
}
// TestDeleteObjectTaggingOnSpecificVersionInVersionedBucket tests deleting tags on a specific version
func TestDeleteObjectTaggingOnSpecificVersionInVersionedBucket(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
// Create two versions with tags
objectKey := "versioned-multi-delete-tags"
version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Body: strings.NewReader("Version 1"),
Key: aws.String(objectKey),
Bucket: aws.String(bucketName),
Tagging: aws.String("version=v1&keep=true"),
})
require.NoError(t, err)
// Small delay to ensure different version IDs
time.Sleep(50 * time.Millisecond)
versionId1 := *version1Resp.VersionId
version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Body: strings.NewReader("Version 2"),
Key: aws.String(objectKey),
Bucket: aws.String(bucketName),
Tagging: aws.String("version=v2&keep=true"),
})
require.NoError(t, err)
versionId2 := *version2Resp.VersionId
// Delete tags from version 1 only
_, err = client.DeleteObjectTagging(context.TODO(), &s3.DeleteObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
})
require.NoError(t, err, "Should be able to delete tags from version 1")
// Verify version 1 has no tags
tagResp1, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
})
require.NoError(t, err)
assert.Len(t, tagResp1.TagSet, 0, "Version 1 should have 0 tags after deletion")
// Verify version 2 still has tags
tagResp2, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
})
require.NoError(t, err)
assert.Len(t, tagResp2.TagSet, 2, "Version 2 should still have 2 tags")
}
// TestGetObjectTaggingOnVersionedBucket tests retrieving tags from versioned objects
func TestGetObjectTaggingOnVersionedBucket(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
// Create multiple versions
objectKey := "versioned-object-get-tags"
version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Body: strings.NewReader("Version 1"),
Key: aws.String(objectKey),
Bucket: aws.String(bucketName),
Tagging: aws.String("v=1&stage=dev"),
})
require.NoError(t, err)
versionId1 := *version1Resp.VersionId
time.Sleep(50 * time.Millisecond)
version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Body: strings.NewReader("Version 2"),
Key: aws.String(objectKey),
Bucket: aws.String(bucketName),
Tagging: aws.String("v=2&stage=prod"),
})
require.NoError(t, err)
versionId2 := *version2Resp.VersionId
// Get tags from specific versions
tagResp1, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
})
require.NoError(t, err, "Should be able to get tags from version 1")
assert.Len(t, tagResp1.TagSet, 2, "Version 1 should have 2 tags")
tagMap1 := make(map[string]string)
for _, tag := range tagResp1.TagSet {
tagMap1[*tag.Key] = *tag.Value
}
assert.Equal(t, "1", tagMap1["v"], "Version 1 should have v=1")
assert.Equal(t, "dev", tagMap1["stage"], "Version 1 should have stage=dev")
tagResp2, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
})
require.NoError(t, err, "Should be able to get tags from version 2")
assert.Len(t, tagResp2.TagSet, 2, "Version 2 should have 2 tags")
tagMap2 := make(map[string]string)
for _, tag := range tagResp2.TagSet {
tagMap2[*tag.Key] = *tag.Value
}
assert.Equal(t, "2", tagMap2["v"], "Version 2 should have v=2")
assert.Equal(t, "prod", tagMap2["stage"], "Version 2 should have stage=prod")
// Get tags from latest version (should be version 2)
tagRespLatest, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err, "Should be able to get tags from latest version")
assert.Len(t, tagRespLatest.TagSet, 2, "Latest version should have 2 tags")
tagMapLatest := make(map[string]string)
for _, tag := range tagRespLatest.TagSet {
tagMapLatest[*tag.Key] = *tag.Value
}
assert.Equal(t, "2", tagMapLatest["v"], "Latest version should have v=2")
assert.Equal(t, "prod", tagMapLatest["stage"], "Latest version should have stage=prod")
}
// TestModifyTagsOnVersionedObject tests changing tags on different versions independently
func TestModifyTagsOnVersionedObject(t *testing.T) {
client := getS3Client(t)
bucketName := createVersionedTestBucket(t, client)
defer cleanupTestBucket(t, client, bucketName)
// Create version 1
objectKey := "versioned-modify-tags"
version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Body: strings.NewReader("Version 1"),
Key: aws.String(objectKey),
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
versionId1 := *version1Resp.VersionId
time.Sleep(50 * time.Millisecond)
// Create version 2
version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
Body: strings.NewReader("Version 2"),
Key: aws.String(objectKey),
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
versionId2 := *version2Resp.VersionId
// Add tags to version 1
_, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
Tagging: &types.Tagging{
TagSet: []types.Tag{
{
Key: aws.String("status"),
Value: aws.String("old"),
},
},
},
})
require.NoError(t, err)
// Modify tags on version 1 (replace status and add new tag)
_, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
Tagging: &types.Tagging{
TagSet: []types.Tag{
{
Key: aws.String("status"),
Value: aws.String("archived"),
},
{
Key: aws.String("archived-date"),
Value: aws.String("2024-01-01"),
},
},
},
})
require.NoError(t, err)
// Add tags to version 2
_, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
Tagging: &types.Tagging{
TagSet: []types.Tag{
{
Key: aws.String("status"),
Value: aws.String("current"),
},
},
},
})
require.NoError(t, err)
// Verify final state
tagResp1, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId1),
})
require.NoError(t, err)
assert.Len(t, tagResp1.TagSet, 2, "Version 1 should have 2 tags after modification")
tagResp2, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
VersionId: aws.String(versionId2),
})
require.NoError(t, err)
assert.Len(t, tagResp2.TagSet, 1, "Version 2 should have 1 tag")
}
// createVersionedTestBucket creates a test bucket with versioning enabled
func createVersionedTestBucket(t *testing.T, client *s3.Client) string {
bucketName := createTestBucket(t, client)
// Enable versioning on the bucket
_, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{
Bucket: aws.String(bucketName),
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
require.NoError(t, err, "Should be able to enable versioning")
// Wait for versioning configuration to be applied
time.Sleep(100 * time.Millisecond)
return bucketName
}
test/s3/versioning/Makefile
+27 -42
View File
@@ -43,13 +43,13 @@ build-weed:
@echo "Building SeaweedFS binary..."
@cd ../../../weed && go build -o weed_binary .
@chmod +x $(WEED_BINARY)
@echo "✅ SeaweedFS binary built at $(WEED_BINARY)"
@echo "OK SeaweedFS binary built at $(WEED_BINARY)"
check-deps: build-weed
@echo "Checking dependencies..."
@echo "🔍 DEBUG: Checking Go installation..."
@echo "DEBUG: Checking Go installation..."
@command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1)
@echo "🔍 DEBUG: Go version: $$(go version)"
@echo "DEBUG: Go version: $$(go version)"
@echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)..."
@test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1)
@echo "🔍 DEBUG: Binary size: $$(ls -lh $(WEED_BINARY) | awk '{print $$5}')"
@@ -57,7 +57,7 @@ check-deps: build-weed
@echo "🔍 DEBUG: Checking Go module dependencies..."
@go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1)
@go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1)
@echo "✅ All dependencies are available"
@echo "OK All dependencies are available"
# Start SeaweedFS server for testing
start-server: check-deps
@@ -81,21 +81,11 @@ start-server: check-deps
@echo "🔍 DEBUG: Creating volume directory..."
@mkdir -p ./test-volume-data
@echo "🔍 DEBUG: Launching SeaweedFS server in background..."
@echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -master.peers=none -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324"
@$(WEED_BINARY) server \
-debug \
-s3 \
-s3.port=$(S3_PORT) \
-s3.allowDeleteBucketNotEmpty=true \
-s3.config=../../../docker/compose/s3.json \
-filer \
-filer.maxMB=64 \
-master.volumeSizeLimitMB=50 \
-master.peers=none \
-volume.max=100 \
@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=./test-volume-data -s3.port=$(S3_PORT) -s3.config=../../../docker/compose/s3.json"
@$(WEED_BINARY) mini \
-dir=./test-volume-data \
-volume.preStopSeconds=1 \
-metricsPort=9324 \
-s3.port=$(S3_PORT) \
-s3.config=../../../docker/compose/s3.json \
> weed-test.log 2>&1 & echo $$! > weed-server.pid
@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
@echo "🔍 DEBUG: Checking if PID is still running..."
@@ -222,13 +212,13 @@ test-with-server: start-server
test-versioning-with-configs: check-deps
@echo "Testing with different S3 configurations..."
@echo "Testing with empty folder allowed..."
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
@$(WEED_BINARY) mini -s3.port=$(S3_PORT) > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
@sleep 5
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
@if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi
@sleep 2
@echo "Testing with delete bucket not empty disabled..."
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
@$(WEED_BINARY) mini -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
@sleep 5
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
@if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi
@@ -271,19 +261,14 @@ debug-server:
@echo "Starting SeaweedFS server in debug mode..."
@$(MAKE) stop-server
@mkdir -p ./test-volume-data
@$(WEED_BINARY) server \
@$(WEED_BINARY) mini \
-debug \
-s3 \
-s3.port=$(S3_PORT) \
-s3.allowDeleteBucketNotEmpty=true \
-s3.config=../../../docker/compose/s3.json \
-filer \
-filer.maxMB=16 \
-master.volumeSizeLimitMB=50 \
-master.peers=none \
-volume.max=100 \
-dir=./test-volume-data \
-volume.preStopSeconds=1 \
-metricsPort=9324
# Run a single test for debugging
@@ -320,24 +305,24 @@ health-check:
# Simple server start without process cleanup (for CI troubleshooting)
start-server-simple: check-deps
@echo "Starting SeaweedFS server (simple mode)..."
@$(WEED_BINARY) server \
-debug \
@$(WEED_BINARY) mini \
-dir=$(SERVER_DIR) \
-s3 \
-s3.port=$(S3_PORT) \
-s3.allowDeleteBucketNotEmpty=true \
-s3.config=../../../docker/compose/s3.json \
-filer \
-filer.maxMB=64 \
-master.volumeSizeLimitMB=50 \
-master.peers=none \
-volume.max=100 \
-volume.preStopSeconds=1 \
-metricsPort=9324 \
> weed-test.log 2>&1 & echo $$! > weed-server.pid
@echo "Server PID: $$(cat weed-server.pid)"
@echo "Waiting for server to start..."
@sleep 10
@curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1 && echo "✅ Server started successfully" || echo "❌ Server failed to start"
-s3.config=$(S3_CONFIG) \
> weed-server.log 2>&1 & \
echo $$! > weed-server.pid
@echo "Waiting for S3 server to be ready..."
@for i in $$(seq 1 30); do \
if echo | nc -z localhost $(S3_PORT); then \
echo "S3 server is ready!"; \
exit 0; \
fi; \
sleep 1; \
done; \
echo "S3 server failed to start"; \
exit 1
# Simple test run without server management
test-versioning-simple: check-deps
weed/admin/Makefile
+1 -3
View File
@@ -160,6 +160,4 @@ $(WEED_BINARY): $(TEMPL_GO_FILES) $(GO_FILES)
# Auto-generate templ files when .templ files change
%_templ.go: %.templ
@echo "Regenerating $@ from $<"
@templ generate
.PHONY: $(TEMPL_GO_FILES)
@templ generate
weed/admin/dash/admin_server.go
+54 -10
View File
@@ -7,6 +7,7 @@ import (
"net/http"
"sort"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
@@ -340,7 +341,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
}
// Get versioning, object lock, and owner information from extended attributes
versioningEnabled := false
versioningStatus := ""
objectLockEnabled := false
objectLockMode := ""
var objectLockDuration int32 = 0
@@ -348,7 +349,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
if resp.Entry.Extended != nil {
// Use shared utility to extract versioning information
versioningEnabled = extractVersioningFromEntry(resp.Entry)
versioningStatus = extractVersioningFromEntry(resp.Entry)
// Use shared utility to extract Object Lock information
objectLockEnabled, objectLockMode, objectLockDuration = extractObjectLockInfoFromEntry(resp.Entry)
@@ -367,7 +368,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
LastModified: time.Unix(resp.Entry.Attributes.Mtime, 0),
Quota: quota,
QuotaEnabled: quotaEnabled,
VersioningEnabled: versioningEnabled,
VersioningStatus: versioningStatus,
ObjectLockEnabled: objectLockEnabled,
ObjectLockMode: objectLockMode,
ObjectLockDuration: objectLockDuration,
@@ -430,7 +431,7 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
details.Bucket.QuotaEnabled = quotaEnabled
// Get versioning, object lock, and owner information from extended attributes
versioningEnabled := false
versioningStatus := ""
objectLockEnabled := false
objectLockMode := ""
var objectLockDuration int32 = 0
@@ -438,7 +439,7 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
if bucketResp.Entry.Extended != nil {
// Use shared utility to extract versioning information
versioningEnabled = extractVersioningFromEntry(bucketResp.Entry)
versioningStatus = extractVersioningFromEntry(bucketResp.Entry)
// Use shared utility to extract Object Lock information
objectLockEnabled, objectLockMode, objectLockDuration = extractObjectLockInfoFromEntry(bucketResp.Entry)
@@ -449,7 +450,7 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error
}
}
details.Bucket.VersioningEnabled = versioningEnabled
details.Bucket.VersioningStatus = versioningStatus
details.Bucket.ObjectLockEnabled = objectLockEnabled
details.Bucket.ObjectLockMode = objectLockMode
details.Bucket.ObjectLockDuration = objectLockDuration
@@ -491,6 +492,45 @@ func (s *AdminServer) listBucketObjects(client filer_pb.SeaweedFilerClient, buck
entry := resp.Entry
if entry.IsDirectory {
// Check if this is a .versions directory (represents a versioned object)
if strings.HasSuffix(entry.Name, ".versions") {
// This directory represents an object, add it as an object without the .versions suffix
objectName := strings.TrimSuffix(entry.Name, ".versions")
objectKey := objectName
if directory != bucketBasePath {
relativePath := directory[len(bucketBasePath)+1:]
objectKey = fmt.Sprintf("%s/%s", relativePath, objectName)
}
// Extract latest version metadata from extended attributes
var size int64 = 0
var mtime int64 = entry.Attributes.Mtime
if entry.Extended != nil {
// Get size of latest version
if sizeBytes, ok := entry.Extended[s3_constants.ExtLatestVersionSizeKey]; ok && len(sizeBytes) == 8 {
size = int64(util.BytesToUint64(sizeBytes))
}
// Get mtime of latest version
if mtimeBytes, ok := entry.Extended[s3_constants.ExtLatestVersionMtimeKey]; ok && len(mtimeBytes) == 8 {
mtime = int64(util.BytesToUint64(mtimeBytes))
}
}
obj := S3Object{
Key: objectKey,
Size: size,
LastModified: time.Unix(mtime, 0),
ETag: "",
StorageClass: "STANDARD",
}
details.Objects = append(details.Objects, obj)
details.TotalCount++
details.TotalSize += size
// Don't recurse into .versions directories
continue
}
// Recursively list subdirectories
subDir := fmt.Sprintf("%s/%s", directory, entry.Name)
err := s.listBucketObjects(client, bucketBasePath, subDir, "", details)
@@ -583,7 +623,7 @@ func (s *AdminServer) DeleteS3Bucket(bucketName string) error {
}
// GetObjectStoreUsers retrieves object store users from identity.json
func (s *AdminServer) GetObjectStoreUsers() ([]ObjectStoreUser, error) {
func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUser, error) {
s3cfg := &iam_pb.S3ApiConfiguration{}
// Load IAM configuration from filer
@@ -616,6 +656,11 @@ func (s *AdminServer) GetObjectStoreUsers() ([]ObjectStoreUser, error) {
continue
}
// Skip service accounts - they should not be parent users
if strings.HasPrefix(identity.Name, serviceAccountPrefix) {
continue
}
user := ObjectStoreUser{
Username: identity.Name,
Permissions: identity.Actions,
@@ -1902,9 +1947,8 @@ func extractObjectLockInfoFromEntry(entry *filer_pb.Entry) (bool, string, int32)
}
// Function to extract versioning information from bucket entry using shared utilities
func extractVersioningFromEntry(entry *filer_pb.Entry) bool {
enabled, _ := s3api.LoadVersioningFromExtended(entry)
return enabled
func extractVersioningFromEntry(entry *filer_pb.Entry) string {
return s3api.GetVersioningStatus(entry)
}
// GetConfigPersistence returns the config persistence manager
weed/admin/dash/auth_middleware.go
+17 -2
View File
@@ -1,6 +1,7 @@
package dash
import (
"crypto/subtle"
"net/http"
"github.com/gin-contrib/sessions"
@@ -25,17 +26,31 @@ func (s *AdminServer) ShowLogin(c *gin.Context) {
}
// HandleLogin handles login form submission
func (s *AdminServer) HandleLogin(username, password string) gin.HandlerFunc {
func (s *AdminServer) HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword string) gin.HandlerFunc {
return func(c *gin.Context) {
loginUsername := c.PostForm("username")
loginPassword := c.PostForm("password")
if loginUsername == username && loginPassword == password {
var role string
var authenticated bool
// Check admin credentials
if adminPassword != "" && loginUsername == adminUser && subtle.ConstantTimeCompare([]byte(loginPassword), []byte(adminPassword)) == 1 {
role = "admin"
authenticated = true
} else if readOnlyPassword != "" && loginUsername == readOnlyUser && subtle.ConstantTimeCompare([]byte(loginPassword), []byte(readOnlyPassword)) == 1 {
// Check read-only credentials
role = "readonly"
authenticated = true
}
if authenticated {
session := sessions.Default(c)
// Clear any existing invalid session data before setting new values
session.Clear()
session.Set("authenticated", true)
session.Set("username", loginUsername)
session.Set("role", role)
if err := session.Save(); err != nil {
// Log the detailed error server-side for diagnostics
glog.Errorf("Failed to save session for user %s: %v", loginUsername, err)
weed/admin/dash/bucket_management.go
+10 -1
View File
@@ -125,6 +125,12 @@ func (s *AdminServer) CreateBucket(c *gin.Context) {
// Convert quota to bytes
quotaBytes := convertQuotaToBytes(req.QuotaSize, req.QuotaUnit)
// Validate quota: if enabled, size must be greater than 0
if req.QuotaEnabled && quotaBytes <= 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "Quota size must be greater than 0 when quota is enabled"})
return
}
// Sanitize owner: trim whitespace and enforce max length
owner := strings.TrimSpace(req.Owner)
if len(owner) > MaxOwnerNameLength {
@@ -466,16 +472,19 @@ func (s *AdminServer) CreateS3BucketWithObjectLock(bucketName string, quotaBytes
// Handle Object Lock configuration using shared utilities
if objectLockEnabled {
var duration int32 = 0
var mode string = ""
if setDefaultRetention {
// Validate Object Lock parameters only when setting default retention
if err := s3api.ValidateObjectLockParameters(objectLockEnabled, objectLockMode, objectLockDuration); err != nil {
return fmt.Errorf("invalid Object Lock parameters: %w", err)
}
duration = objectLockDuration
mode = objectLockMode
}
// Create Object Lock configuration using shared utility
objectLockConfig := s3api.CreateObjectLockConfigurationFromParams(objectLockEnabled, objectLockMode, duration)
objectLockConfig := s3api.CreateObjectLockConfigurationFromParams(objectLockEnabled, mode, duration)
// Store Object Lock configuration in extended attributes using shared utility
if err := s3api.StoreObjectLockConfigurationInExtended(bucketEntry, objectLockConfig); err != nil {
weed/admin/dash/file_browser_data.go
+144 -130
View File
@@ -2,8 +2,7 @@ package dash
import (
"context"
"path/filepath"
"sort"
"path"
"strings"
"time"
@@ -34,34 +33,49 @@ type BreadcrumbItem struct {
// FileBrowserData contains all data needed for the file browser view
type FileBrowserData struct {
Username string `json:"username"`
CurrentPath string `json:"current_path"`
ParentPath string `json:"parent_path"`
Breadcrumbs []BreadcrumbItem `json:"breadcrumbs"`
Entries []FileEntry `json:"entries"`
TotalEntries int `json:"total_entries"`
TotalSize int64 `json:"total_size"`
LastUpdated time.Time `json:"last_updated"`
IsBucketPath bool `json:"is_bucket_path"`
BucketName string `json:"bucket_name"`
Username string `json:"username"`
CurrentPath string `json:"current_path"`
ParentPath string `json:"parent_path"`
Breadcrumbs []BreadcrumbItem `json:"breadcrumbs"`
Entries []FileEntry `json:"entries"`
LastUpdated time.Time `json:"last_updated"`
IsBucketPath bool `json:"is_bucket_path"`
BucketName string `json:"bucket_name"`
// Pagination fields
PageSize int `json:"page_size"`
HasNextPage bool `json:"has_next_page"`
LastFileName string `json:"last_file_name"` // Cursor for next page
CurrentLastFileName string `json:"current_last_file_name"` // Cursor from current request (for page size changes)
}
// GetFileBrowser retrieves file browser data for a given path
func (s *AdminServer) GetFileBrowser(path string) (*FileBrowserData, error) {
if path == "" {
path = "/"
// GetFileBrowser retrieves file browser data for a given path with cursor-based pagination
func (s *AdminServer) GetFileBrowser(dir string, lastFileName string, pageSize int) (*FileBrowserData, error) {
if dir == "" {
dir = "/"
}
// Set defaults for pagination
if pageSize < 1 {
pageSize = 20 // Default page size
}
var entries []FileEntry
var totalSize int64
// Get directory listing from filer
// Fetch entries using cursor-based pagination
// We fetch pageSize+1 to determine if there's a next page
fetchLimit := pageSize + 1
var fetchedCount int
var lastEntryName string
err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
// Fetch entries starting from the cursor (lastFileName)
stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{
Directory: path,
Directory: dir,
Prefix: "",
Limit: 1000,
InclusiveStartFrom: false,
Limit: uint32(fetchLimit),
StartFromFileName: lastFileName,
InclusiveStartFrom: false, // Don't include the cursor file itself
})
if err != nil {
return err
@@ -81,101 +95,102 @@ func (s *AdminServer) GetFileBrowser(path string) (*FileBrowserData, error) {
continue
}
fullPath := path
if !strings.HasSuffix(fullPath, "/") {
fullPath += "/"
}
fullPath += entry.Name
fetchedCount++
var modTime time.Time
if entry.Attributes != nil && entry.Attributes.Mtime > 0 {
modTime = time.Unix(entry.Attributes.Mtime, 0)
}
// Only add entries up to pageSize (the +1 is just to check for next page)
if fetchedCount <= pageSize {
fullPath := path.Join(dir, entry.Name)
var mode string
var uid, gid uint32
var size int64
var replication, collection string
var ttlSec int32
if entry.Attributes != nil {
mode = FormatFileMode(entry.Attributes.FileMode)
uid = entry.Attributes.Uid
gid = entry.Attributes.Gid
size = int64(entry.Attributes.FileSize)
ttlSec = entry.Attributes.TtlSec
}
// Get replication and collection from entry extended attributes or chunks
if entry.Extended != nil {
if repl, ok := entry.Extended["replication"]; ok {
replication = string(repl)
var modTime time.Time
if entry.Attributes != nil && entry.Attributes.Mtime > 0 {
modTime = time.Unix(entry.Attributes.Mtime, 0)
}
if coll, ok := entry.Extended["collection"]; ok {
collection = string(coll)
var mode string
var uid, gid uint32
var size int64
var replication, collection string
var ttlSec int32
if entry.Attributes != nil {
mode = FormatFileMode(entry.Attributes.FileMode)
uid = entry.Attributes.Uid
gid = entry.Attributes.Gid
size = int64(entry.Attributes.FileSize)
ttlSec = entry.Attributes.TtlSec
}
}
// Determine MIME type based on file extension
mime := "application/octet-stream"
if entry.IsDirectory {
mime = "inode/directory"
} else {
ext := strings.ToLower(filepath.Ext(entry.Name))
switch ext {
case ".txt", ".log":
mime = "text/plain"
case ".html", ".htm":
mime = "text/html"
case ".css":
mime = "text/css"
case ".js":
mime = "application/javascript"
case ".json":
mime = "application/json"
case ".xml":
mime = "application/xml"
case ".pdf":
mime = "application/pdf"
case ".jpg", ".jpeg":
mime = "image/jpeg"
case ".png":
mime = "image/png"
case ".gif":
mime = "image/gif"
case ".svg":
mime = "image/svg+xml"
case ".mp4":
mime = "video/mp4"
case ".mp3":
mime = "audio/mpeg"
case ".zip":
mime = "application/zip"
case ".tar":
mime = "application/x-tar"
case ".gz":
mime = "application/gzip"
// Get replication and collection from entry extended attributes
if entry.Extended != nil {
if repl, ok := entry.Extended["replication"]; ok {
replication = string(repl)
}
if coll, ok := entry.Extended["collection"]; ok {
collection = string(coll)
}
}
}
fileEntry := FileEntry{
Name: entry.Name,
FullPath: fullPath,
IsDirectory: entry.IsDirectory,
Size: size,
ModTime: modTime,
Mode: mode,
Uid: uid,
Gid: gid,
Mime: mime,
Replication: replication,
Collection: collection,
TtlSec: ttlSec,
}
// Determine MIME type based on file extension
mime := "application/octet-stream"
if entry.IsDirectory {
mime = "inode/directory"
} else {
ext := strings.ToLower(path.Ext(entry.Name))
switch ext {
case ".txt", ".log":
mime = "text/plain"
case ".html", ".htm":
mime = "text/html"
case ".css":
mime = "text/css"
case ".js":
mime = "application/javascript"
case ".json":
mime = "application/json"
case ".xml":
mime = "application/xml"
case ".pdf":
mime = "application/pdf"
case ".jpg", ".jpeg":
mime = "image/jpeg"
case ".png":
mime = "image/png"
case ".gif":
mime = "image/gif"
case ".svg":
mime = "image/svg+xml"
case ".mp4":
mime = "video/mp4"
case ".mp3":
mime = "audio/mpeg"
case ".zip":
mime = "application/zip"
case ".tar":
mime = "application/x-tar"
case ".gz":
mime = "application/gzip"
}
}
fileEntry := FileEntry{
Name: entry.Name,
FullPath: fullPath,
IsDirectory: entry.IsDirectory,
Size: size,
ModTime: modTime,
Mode: mode,
Uid: uid,
Gid: gid,
Mime: mime,
Replication: replication,
Collection: collection,
TtlSec: ttlSec,
}
entries = append(entries, fileEntry)
lastEntryName = entry.Name
entries = append(entries, fileEntry)
if !entry.IsDirectory {
totalSize += size
}
}
@@ -186,21 +201,16 @@ func (s *AdminServer) GetFileBrowser(path string) (*FileBrowserData, error) {
return nil, err
}
// Sort entries: directories first, then files, both alphabetically
sort.Slice(entries, func(i, j int) bool {
if entries[i].IsDirectory != entries[j].IsDirectory {
return entries[i].IsDirectory
}
return strings.ToLower(entries[i].Name) < strings.ToLower(entries[j].Name)
})
// Determine if there's a next page
hasNextPage := fetchedCount > pageSize
// Generate breadcrumbs
breadcrumbs := s.generateBreadcrumbs(path)
breadcrumbs := s.generateBreadcrumbs(dir)
// Calculate parent path
parentPath := "/"
if path != "/" {
parentPath = filepath.Dir(path)
if dir != "/" {
parentPath = path.Dir(dir)
if parentPath == "." {
parentPath = "/"
}
@@ -209,29 +219,33 @@ func (s *AdminServer) GetFileBrowser(path string) (*FileBrowserData, error) {
// Check if this is a bucket path
isBucketPath := false
bucketName := ""
if strings.HasPrefix(path, "/buckets/") {
if strings.HasPrefix(dir, "/buckets/") {
isBucketPath = true
pathParts := strings.Split(strings.Trim(path, "/"), "/")
pathParts := strings.Split(strings.Trim(dir, "/"), "/")
if len(pathParts) >= 2 {
bucketName = pathParts[1]
}
}
return &FileBrowserData{
CurrentPath: path,
ParentPath: parentPath,
Breadcrumbs: breadcrumbs,
Entries: entries,
TotalEntries: len(entries),
TotalSize: totalSize,
CurrentPath: dir,
ParentPath: parentPath,
Breadcrumbs: breadcrumbs,
Entries: entries,
LastUpdated: time.Now(),
IsBucketPath: isBucketPath,
BucketName: bucketName,
// Pagination metadata
PageSize: pageSize,
HasNextPage: hasNextPage,
LastFileName: lastEntryName, // Store for next page navigation
CurrentLastFileName: lastFileName, // Store input cursor for page size changes
}, nil
}
// generateBreadcrumbs creates breadcrumb navigation for the current path
func (s *AdminServer) generateBreadcrumbs(path string) []BreadcrumbItem {
func (s *AdminServer) generateBreadcrumbs(dir string) []BreadcrumbItem {
var breadcrumbs []BreadcrumbItem
// Always start with root
@@ -240,12 +254,12 @@ func (s *AdminServer) generateBreadcrumbs(path string) []BreadcrumbItem {
Path: "/",
})
if path == "/" {
if dir == "/" {
return breadcrumbs
}
// Split path and build breadcrumbs
parts := strings.Split(strings.Trim(path, "/"), "/")
parts := strings.Split(strings.Trim(dir, "/"), "/")
currentPath := ""
for _, part := range parts {
@@ -258,7 +272,7 @@ func (s *AdminServer) generateBreadcrumbs(path string) []BreadcrumbItem {
displayName := part
if len(breadcrumbs) == 1 && part == "buckets" {
displayName = "Object Store Buckets"
} else if len(breadcrumbs) == 2 && strings.HasPrefix(path, "/buckets/") {
} else if len(breadcrumbs) == 2 && strings.HasPrefix(dir, "/buckets/") {
displayName = "📦 " + part // Add bucket icon to bucket name
}
weed/admin/dash/file_browser_data_test.go
+502
View File
@@ -0,0 +1,502 @@
package dash
import (
"path"
"strings"
"testing"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// TestGenerateBreadcrumbs tests the actual breadcrumb generation function
// from the production code with various path scenarios
func TestGenerateBreadcrumbs(t *testing.T) {
s := &AdminServer{}
tests := []struct {
name string
path string
expected []BreadcrumbItem
}{
{
name: "root path",
path: "/",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
},
},
{
name: "simple path",
path: "/folder",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
{Name: "folder", Path: "/folder"},
},
},
{
name: "nested path",
path: "/folder/subfolder",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
{Name: "folder", Path: "/folder"},
{Name: "subfolder", Path: "/folder/subfolder"},
},
},
{
name: "bucket path",
path: "/buckets/mybucket",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
{Name: "Object Store Buckets", Path: "/buckets"},
{Name: "📦 mybucket", Path: "/buckets/mybucket"},
},
},
{
name: "bucket nested path",
path: "/buckets/mybucket/folder",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
{Name: "Object Store Buckets", Path: "/buckets"},
{Name: "📦 mybucket", Path: "/buckets/mybucket"},
{Name: "folder", Path: "/buckets/mybucket/folder"},
},
},
{
name: "path with trailing slash",
path: "/folder/",
expected: []BreadcrumbItem{
{Name: "Root", Path: "/"},
{Name: "folder", Path: "/folder"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Call the actual production function
result := s.generateBreadcrumbs(tt.path)
if len(result) != len(tt.expected) {
t.Errorf("expected %d breadcrumbs, got %d", len(tt.expected), len(result))
return
}
for i, crumb := range result {
if crumb.Name != tt.expected[i].Name {
t.Errorf("breadcrumb %d: expected name %q, got %q", i, tt.expected[i].Name, crumb.Name)
}
if crumb.Path != tt.expected[i].Path {
t.Errorf("breadcrumb %d: expected path %q, got %q", i, tt.expected[i].Path, crumb.Path)
}
}
})
}
}
// TestPathHandlingWithForwardSlashes verifies that the production code
// correctly handles paths with forward slashes (not OS-specific backslashes)
func TestPathHandlingWithForwardSlashes(t *testing.T) {
tests := []struct {
name string
path string
hasSlash bool
}{
{
name: "root",
path: "/",
hasSlash: true,
},
{
name: "single level",
path: "/test",
hasSlash: true,
},
{
name: "multiple levels",
path: "/a/b/c",
hasSlash: true,
},
{
name: "bucket path",
path: "/buckets/mybucket/file",
hasSlash: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Verify no backslashes appear in paths (which would be wrong on Windows)
if strings.Contains(tt.path, "\\") {
t.Errorf("path contains backslash: %q", tt.path)
}
// Verify forward slashes are used
if tt.hasSlash && !strings.Contains(tt.path, "/") {
t.Errorf("path should contain forward slash: %q", tt.path)
}
})
}
}
// TestParentPathCalculationLogic verifies that parent path calculation
// uses path.Dir semantics (forward slashes), not filepath.Dir (OS-specific)
func TestParentPathCalculationLogic(t *testing.T) {
tests := []struct {
name string
currentDir string
expected string
}{
{
name: "root path",
currentDir: "/",
expected: "/",
},
{
name: "single level",
currentDir: "/folder",
expected: "/",
},
{
name: "two levels",
currentDir: "/folder/subfolder",
expected: "/folder",
},
{
name: "deep nesting",
currentDir: "/a/b/c/d",
expected: "/a/b/c",
},
{
name: "bucket root",
currentDir: "/buckets",
expected: "/",
},
{
name: "bucket directory",
currentDir: "/buckets/mybucket",
expected: "/buckets",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Verify using path.Dir (the correct approach for URLs)
// This demonstrates the expected behavior
parentPath := "/"
if tt.currentDir != "/" {
// path.Dir always uses forward slashes
parentPath = path.Dir(tt.currentDir)
if parentPath == "." {
parentPath = "/"
}
}
if parentPath != tt.expected {
t.Errorf("expected parent %q, got %q for %q", tt.expected, parentPath, tt.currentDir)
}
// Verify no backslashes in the result
if strings.Contains(parentPath, "\\") {
t.Errorf("parent path contains backslash: %q", parentPath)
}
})
}
}
// TestFileExtensionHandlingLogic verifies that file extensions are correctly
// identified using path semantics (always forward slashes)
func TestFileExtensionHandlingLogic(t *testing.T) {
tests := []struct {
filename string
expected string
}{
{"file.txt", ".txt"},
{"file.log", ".log"},
{"archive.tar.gz", ".gz"},
{"image.jpg", ".jpg"},
{"document.pdf", ".pdf"},
{"data.json", ".json"},
{"noextension", ""},
{".hidden", ".hidden"},
{"file.TXT", ".txt"},
{"file.JPG", ".jpg"},
}
for _, tt := range tests {
t.Run(tt.filename, func(t *testing.T) {
// Verify using path.Ext + strings.ToLower (the correct approach)
ext := strings.ToLower(path.Ext(tt.filename))
if ext != tt.expected {
t.Errorf("expected extension %q for %q, got %q", tt.expected, tt.filename, ext)
}
})
}
}
// TestBucketPathDetectionLogic verifies bucket path detection logic
func TestBucketPathDetectionLogic(t *testing.T) {
tests := []struct {
name string
path string
isBucket bool
expectedName string
}{
{
name: "root is not a bucket path",
path: "/",
isBucket: false,
expectedName: "",
},
{
name: "buckets root",
path: "/buckets/",
isBucket: true,
expectedName: "",
},
{
name: "single bucket",
path: "/buckets/mybucket",
isBucket: true,
expectedName: "mybucket",
},
{
name: "bucket with nested path",
path: "/buckets/mybucket/folder/file",
isBucket: true,
expectedName: "mybucket",
},
{
name: "non-bucket path",
path: "/data/folder",
isBucket: false,
expectedName: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Verify the bucket path detection logic
isBucketPath := false
bucketName := ""
if strings.HasPrefix(tt.path, "/buckets/") {
isBucketPath = true
pathParts := strings.Split(strings.Trim(tt.path, "/"), "/")
if len(pathParts) >= 2 {
bucketName = pathParts[1]
}
}
if isBucketPath != tt.isBucket {
t.Errorf("expected isBucketPath=%v, got %v for path %q", tt.isBucket, isBucketPath, tt.path)
}
if bucketName != tt.expectedName {
t.Errorf("expected bucket name %q, got %q for path %q", tt.expectedName, bucketName, tt.path)
}
})
}
}
// TestPathJoinHandlesEdgeCases verifies that path.Join handles edge cases
// properly for URL path construction (unlike filepath.Join which is OS-specific)
func TestPathJoinHandlesEdgeCases(t *testing.T) {
tests := []struct {
testName string
dir string
filename string
expected string
}{
{
testName: "root directory",
dir: "/",
filename: "file.txt",
expected: "/file.txt",
},
{
testName: "simple directory",
dir: "/folder",
filename: "file.txt",
expected: "/folder/file.txt",
},
{
testName: "nested directory",
dir: "/a/b/c",
filename: "file.txt",
expected: "/a/b/c/file.txt",
},
{
testName: "handles trailing slash",
dir: "/folder/",
filename: "file.txt",
expected: "/folder/file.txt",
},
{
testName: "handles empty name",
dir: "/folder",
filename: "",
expected: "/folder",
},
}
for _, tt := range tests {
t.Run(tt.testName, func(t *testing.T) {
// Verify path.Join behavior for URL paths
result := path.Join(tt.dir, tt.filename)
if result != tt.expected {
t.Errorf("path.Join(%q, %q) = %q, expected %q", tt.dir, tt.filename, result, tt.expected)
}
// Verify no backslashes in the result
if strings.Contains(result, "\\") {
t.Errorf("result contains backslash: %q", result)
}
})
}
}
// TestWindowsPathNormalizationBehavior validates that Windows-style paths
// are correctly converted to forward slashes for URL compatibility.
// This test verifies the actual util.CleanWindowsPath() function used in
// the ShowFileBrowser handler.
func TestWindowsPathNormalizationBehavior(t *testing.T) {
tests := []struct {
name string
windowsPath string
expectedNormPath string
}{
{
name: "backslash separator",
windowsPath: "\\folder\\subfolder",
expectedNormPath: "/folder/subfolder",
},
{
name: "mixed separators",
windowsPath: "/folder\\subfolder/file",
expectedNormPath: "/folder/subfolder/file",
},
{
name: "already normalized",
windowsPath: "/folder/file",
expectedNormPath: "/folder/file",
},
{
name: "simple backslash path",
windowsPath: "\\data",
expectedNormPath: "/data",
},
{
name: "deep nested path",
windowsPath: "\\a\\b\\c\\d",
expectedNormPath: "/a/b/c/d",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Test the actual production function
normalized := util.CleanWindowsPath(tt.windowsPath)
if normalized != tt.expectedNormPath {
t.Errorf("CleanWindowsPath(%q): expected %q, got %q",
tt.windowsPath, tt.expectedNormPath, normalized)
}
})
}
}
// TestBreadcrumbPathFormatting validates that breadcrumb paths always
// use forward slashes and maintain proper URL format
func TestBreadcrumbPathFormatting(t *testing.T) {
s := &AdminServer{}
testPaths := []string{
"/",
"/folder",
"/folder/subfolder",
"/buckets/mybucket",
"/buckets/mybucket/data",
}
for _, testPath := range testPaths {
t.Run("breadcrumbs_for_"+testPath, func(t *testing.T) {
breadcrumbs := s.generateBreadcrumbs(testPath)
// Verify all breadcrumb paths use forward slashes
for i, crumb := range breadcrumbs {
if strings.Contains(crumb.Path, "\\") {
t.Errorf("breadcrumb %d has backslash in path: %q", i, crumb.Path)
}
// Verify paths start with / (except when empty)
if crumb.Path != "" && !strings.HasPrefix(crumb.Path, "/") {
t.Errorf("breadcrumb %d path should start with /: %q", i, crumb.Path)
}
}
})
}
}
// TestDirectoryNavigation validates the complete navigation flow
// for various path scenarios
func TestDirectoryNavigation(t *testing.T) {
s := &AdminServer{}
tests := []struct {
name string
currentPath string
expectedParent string
expectedCrumbs int
}{
{
name: "navigate from root",
currentPath: "/",
expectedParent: "/",
expectedCrumbs: 1,
},
{
name: "navigate from single folder",
currentPath: "/documents",
expectedParent: "/",
expectedCrumbs: 2,
},
{
name: "navigate from nested folder",
currentPath: "/documents/projects/current",
expectedParent: "/documents/projects",
expectedCrumbs: 4,
},
{
name: "navigate bucket contents",
currentPath: "/buckets/data/files",
expectedParent: "/buckets/data",
expectedCrumbs: 4,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Verify breadcrumbs are generated correctly
breadcrumbs := s.generateBreadcrumbs(tt.currentPath)
if len(breadcrumbs) != tt.expectedCrumbs {
t.Errorf("expected %d breadcrumbs, got %d", tt.expectedCrumbs, len(breadcrumbs))
}
// Verify parent path calculation
expectedParent := "/"
if tt.currentPath != "/" {
expectedParent = path.Dir(tt.currentPath)
if expectedParent == "." {
expectedParent = "/"
}
}
if expectedParent != tt.expectedParent {
t.Errorf("expected parent %q, got %q", tt.expectedParent, expectedParent)
}
// Verify all paths use forward slashes
for i, crumb := range breadcrumbs {
if strings.Contains(crumb.Path, "\\") {
t.Errorf("breadcrumb %d contains backslash: %q", i, crumb.Path)
}
}
})
}
}
weed/admin/dash/middleware.go
+47 -4
View File
@@ -2,17 +2,30 @@ package dash
import (
"net/http"
"strings"
"github.com/gin-contrib/sessions"
"github.com/gin-gonic/gin"
)
// setAuthContext sets username and role in context for use in handlers
func setAuthContext(c *gin.Context, username, role interface{}) {
c.Set("username", username)
if role != nil {
c.Set("role", role)
} else {
// Default to admin for backward compatibility
c.Set("role", "admin")
}
}
// RequireAuth checks if user is authenticated
func RequireAuth() gin.HandlerFunc {
return func(c *gin.Context) {
session := sessions.Default(c)
authenticated := session.Get("authenticated")
username := session.Get("username")
role := session.Get("role")
if authenticated != true || username == nil {
c.Redirect(http.StatusTemporaryRedirect, "/login")
@@ -20,8 +33,8 @@ func RequireAuth() gin.HandlerFunc {
return
}
// Set username in context for use in handlers
c.Set("username", username)
// Set username and role in context for use in handlers
setAuthContext(c, username, role)
c.Next()
}
}
@@ -33,6 +46,7 @@ func RequireAuthAPI() gin.HandlerFunc {
session := sessions.Default(c)
authenticated := session.Get("authenticated")
username := session.Get("username")
role := session.Get("role")
if authenticated != true || username == nil {
c.JSON(http.StatusUnauthorized, gin.H{
@@ -43,8 +57,37 @@ func RequireAuthAPI() gin.HandlerFunc {
return
}
// Set username in context for use in handlers
c.Set("username", username)
// Set username and role in context for use in handlers
setAuthContext(c, username, role)
c.Next()
}
}
// RequireWriteAccess checks if user has admin role (write access)
// Returns JSON error for API endpoints, redirects for HTML endpoints
func RequireWriteAccess() gin.HandlerFunc {
return func(c *gin.Context) {
role, exists := c.Get("role")
if !exists {
role = "admin" // Default for backward compatibility
}
roleStr, ok := role.(string)
if !ok || roleStr != "admin" {
// Check if this is an API request (path starts with /api) or HTML request
path := c.Request.URL.Path
if strings.HasPrefix(path, "/api") {
c.JSON(http.StatusForbidden, gin.H{
"error": "Insufficient permissions",
"message": "This operation requires admin access. Read-only users can only view data.",
})
} else {
c.Redirect(http.StatusSeeOther, "/admin?error=Insufficient permissions")
}
c.Abort()
return
}
c.Next()
}
}
weed/admin/dash/service_account_helpers.go
+53
View File
@@ -0,0 +1,53 @@
package dash
import (
"fmt"
"strings"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
)
// identityToServiceAccount converts an IAM identity to a ServiceAccount struct
// This helper reduces code duplication across GetServiceAccounts, GetServiceAccountDetails,
// UpdateServiceAccount, and GetServiceAccountByAccessKey
func identityToServiceAccount(identity *iam_pb.Identity) (*ServiceAccount, error) {
if identity == nil {
return nil, fmt.Errorf("identity cannot be nil")
}
if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) {
return nil, fmt.Errorf("not a service account: %s", identity.GetName())
}
parts := strings.SplitN(identity.GetName(), ":", 3)
if len(parts) < 3 {
return nil, fmt.Errorf("invalid service account ID format")
}
sa := &ServiceAccount{
ID: identity.GetName(),
ParentUser: parts[1],
Status: StatusActive,
CreateDate: getCreationDate(identity.GetActions()),
Expiration: getExpiration(identity.GetActions()),
}
// Get description from account display name
if identity.Account != nil {
sa.Description = identity.Account.GetDisplayName()
}
// Get access key from credentials
if len(identity.Credentials) > 0 {
sa.AccessKeyId = identity.Credentials[0].GetAccessKey()
}
// Check if disabled
for _, action := range identity.GetActions() {
if action == disabledAction {
sa.Status = StatusInactive
break
}
}
return sa, nil
}
weed/admin/dash/service_account_management.go
+434
View File
@@ -0,0 +1,434 @@
package dash
import (
"context"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
)
var (
// ErrServiceAccountNotFound is returned when a service account is not found
ErrServiceAccountNotFound = errors.New("service account not found")
)
const (
createdAtActionPrefix = "createdAt:"
expirationActionPrefix = "expiresAt:"
disabledAction = "__disabled__"
serviceAccountPrefix = "sa:"
accessKeyPrefix = "ABIA" // Service account access keys use ABIA prefix
// Status constants
StatusActive = "Active"
StatusInactive = "Inactive"
)
// Helper functions for managing creation timestamps in actions
func getCreationDate(actions []string) time.Time {
for _, action := range actions {
if strings.HasPrefix(action, createdAtActionPrefix) {
timestampStr := strings.TrimPrefix(action, createdAtActionPrefix)
if timestamp, err := strconv.ParseInt(timestampStr, 10, 64); err == nil {
return time.Unix(timestamp, 0)
}
}
}
return time.Time{} // Return zero time for legacy service accounts without stored creation date
}
func setCreationDate(actions []string, createDate time.Time) []string {
// Remove any existing createdAt action
filtered := make([]string, 0, len(actions)+1)
for _, action := range actions {
if !strings.HasPrefix(action, createdAtActionPrefix) {
filtered = append(filtered, action)
}
}
// Add new createdAt action
filtered = append(filtered, fmt.Sprintf("%s%d", createdAtActionPrefix, createDate.Unix()))
return filtered
}
// Helper functions for managing expiration timestamps in actions
func getExpiration(actions []string) time.Time {
for _, action := range actions {
if strings.HasPrefix(action, expirationActionPrefix) {
timestampStr := strings.TrimPrefix(action, expirationActionPrefix)
if timestamp, err := strconv.ParseInt(timestampStr, 10, 64); err == nil {
return time.Unix(timestamp, 0)
}
}
}
return time.Time{} // No expiration set
}
func setExpiration(actions []string, expiration time.Time) []string {
// Remove any existing expiration action
filtered := make([]string, 0, len(actions)+1)
for _, action := range actions {
if !strings.HasPrefix(action, expirationActionPrefix) {
filtered = append(filtered, action)
}
}
// Add new expiration action if not zero
if !expiration.IsZero() {
filtered = append(filtered, fmt.Sprintf("%s%d", expirationActionPrefix, expiration.Unix()))
}
return filtered
}
// GetServiceAccounts returns all service accounts, optionally filtered by parent user
// NOTE: Service accounts are stored as special identities with "sa:" prefix
func (s *AdminServer) GetServiceAccounts(ctx context.Context, parentUser string) ([]ServiceAccount, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
// Load the current configuration to find service account identities
config, err := s.credentialManager.LoadConfiguration(ctx)
if err != nil {
return nil, fmt.Errorf("failed to load configuration: %w", err)
}
var accounts []ServiceAccount
// Service accounts are stored as identities with "sa:" prefix in their name
// Format: "sa:<parent_user>:<uuid>"
for _, identity := range config.GetIdentities() {
if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) {
continue
}
parts := strings.SplitN(identity.GetName(), ":", 3)
if len(parts) < 3 {
continue
}
parent := parts[1]
saId := identity.GetName()
// Filter by parent user if specified
if parentUser != "" && parent != parentUser {
continue
}
// Extract description from account display name if available
description := ""
status := StatusActive
if identity.Account != nil {
description = identity.Account.GetDisplayName()
}
// Get access key from credentials
accessKey := ""
if len(identity.Credentials) > 0 {
accessKey = identity.Credentials[0].GetAccessKey()
// Service accounts use ABIA prefix
if !strings.HasPrefix(accessKey, accessKeyPrefix) {
continue // Not a service account
}
}
// Check if disabled (stored in actions)
for _, action := range identity.GetActions() {
if action == disabledAction {
status = StatusInactive
break
}
}
accounts = append(accounts, ServiceAccount{
ID: saId,
ParentUser: parent,
Description: description,
AccessKeyId: accessKey,
Status: status,
CreateDate: getCreationDate(identity.GetActions()),
Expiration: getExpiration(identity.GetActions()),
})
}
return accounts, nil
}
// GetServiceAccountDetails returns detailed information about a specific service account
func (s *AdminServer) GetServiceAccountDetails(ctx context.Context, id string) (*ServiceAccount, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
// Get the identity
identity, err := s.credentialManager.GetUser(ctx, id)
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrServiceAccountNotFound, id)
}
if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) {
return nil, fmt.Errorf("%w: not a service account: %s", ErrServiceAccountNotFound, id)
}
parts := strings.SplitN(identity.GetName(), ":", 3)
if len(parts) < 3 {
return nil, fmt.Errorf("invalid service account ID format")
}
account := &ServiceAccount{
ID: id,
ParentUser: parts[1],
Status: StatusActive,
CreateDate: getCreationDate(identity.GetActions()),
Expiration: getExpiration(identity.GetActions()),
}
if identity.Account != nil {
account.Description = identity.Account.GetDisplayName()
}
if len(identity.Credentials) > 0 {
account.AccessKeyId = identity.Credentials[0].GetAccessKey()
}
// Check if disabled
for _, action := range identity.GetActions() {
if action == disabledAction {
account.Status = StatusInactive
break
}
}
return account, nil
}
// CreateServiceAccount creates a new service account for a parent user
func (s *AdminServer) CreateServiceAccount(ctx context.Context, req CreateServiceAccountRequest) (*ServiceAccount, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
// Validate parent user exists
_, err := s.credentialManager.GetUser(ctx, req.ParentUser)
if err != nil {
return nil, fmt.Errorf("parent user not found: %s", req.ParentUser)
}
// Generate unique ID and credentials
uuid := generateAccountId()
saId := fmt.Sprintf("sa:%s:%s", req.ParentUser, uuid)
accessKey := accessKeyPrefix + generateAccessKey()[len(accessKeyPrefix):] // Use ABIA prefix for service accounts
secretKey := generateSecretKey()
// Create the service account as a special identity
now := time.Now()
// Parse expiration if provided
var expiration time.Time
if req.Expiration != "" {
var err error
expiration, err = time.Parse(time.RFC3339, req.Expiration)
if err != nil {
return nil, fmt.Errorf("invalid expiration format: %w", err)
}
}
identity := &iam_pb.Identity{
Name: saId,
Account: &iam_pb.Account{
Id: uuid,
DisplayName: req.Description,
},
Credentials: []*iam_pb.Credential{
{
AccessKey: accessKey,
SecretKey: secretKey,
},
},
// Store creation date and expiration in actions
Actions: setExpiration(setCreationDate([]string{}, now), expiration),
}
// Create the service account
err = s.credentialManager.CreateUser(ctx, identity)
if err != nil {
return nil, fmt.Errorf("failed to create service account: %w", err)
}
glog.V(1).Infof("Created service account %s for user %s", saId, req.ParentUser)
return &ServiceAccount{
ID: saId,
ParentUser: req.ParentUser,
Description: req.Description,
AccessKeyId: accessKey,
SecretAccessKey: secretKey, // Only returned on creation
Status: StatusActive,
CreateDate: now,
Expiration: expiration,
}, nil
}
// UpdateServiceAccount updates an existing service account
func (s *AdminServer) UpdateServiceAccount(ctx context.Context, id string, req UpdateServiceAccountRequest) (*ServiceAccount, error) {
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
// Get existing identity
identity, err := s.credentialManager.GetUser(ctx, id)
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrServiceAccountNotFound, id)
}
if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) {
return nil, fmt.Errorf("%w: not a service account: %s", ErrServiceAccountNotFound, id)
}
// Update description if provided
if req.Description != "" {
if identity.Account == nil {
identity.Account = &iam_pb.Account{}
}
identity.Account.DisplayName = req.Description
}
// Update status by adding/removing disabled action
if req.Status != "" {
// Remove existing disabled marker
newActions := make([]string, 0, len(identity.Actions))
for _, action := range identity.Actions {
if action != disabledAction {
newActions = append(newActions, action)
}
}
// Add disabled action if setting to Inactive
if req.Status == StatusInactive {
newActions = append(newActions, disabledAction)
}
identity.Actions = newActions
}
// Update expiration if provided
if req.Expiration != "" {
var expiration time.Time
var err error
expiration, err = time.Parse(time.RFC3339, req.Expiration)
if err != nil {
return nil, fmt.Errorf("invalid expiration format: %w", err)
}
identity.Actions = setExpiration(identity.Actions, expiration)
}
// Update the identity
err = s.credentialManager.UpdateUser(ctx, id, identity)
if err != nil {
return nil, fmt.Errorf("failed to update service account: %w", err)
}
glog.V(1).Infof("Updated service account %s", id)
// Build response
parts := strings.SplitN(id, ":", 3)
if len(parts) < 3 {
return nil, fmt.Errorf("invalid service account ID format")
}
result := &ServiceAccount{
ID: id,
ParentUser: parts[1],
Description: identity.Account.GetDisplayName(),
Status: StatusActive,
CreateDate: getCreationDate(identity.Actions),
}
if len(identity.Credentials) > 0 {
result.AccessKeyId = identity.Credentials[0].GetAccessKey()
}
for _, action := range identity.Actions {
if action == disabledAction {
result.Status = StatusInactive
break
}
}
return result, nil
}
// DeleteServiceAccount deletes a service account
func (s *AdminServer) DeleteServiceAccount(ctx context.Context, id string) error {
if s.credentialManager == nil {
return fmt.Errorf("credential manager not available")
}
// Verify it's a service account
identity, err := s.credentialManager.GetUser(ctx, id)
if err != nil {
return fmt.Errorf("%w: %s", ErrServiceAccountNotFound, id)
}
if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) {
return fmt.Errorf("%w: not a service account: %s", ErrServiceAccountNotFound, id)
}
// Delete the identity
err = s.credentialManager.DeleteUser(ctx, id)
if err != nil {
return fmt.Errorf("failed to delete service account: %w", err)
}
glog.V(1).Infof("Deleted service account %s", id)
return nil
}
// GetServiceAccountByAccessKey finds a service account by its access key
func (s *AdminServer) GetServiceAccountByAccessKey(ctx context.Context, accessKey string) (*ServiceAccount, error) {
if !strings.HasPrefix(accessKey, accessKeyPrefix) {
return nil, fmt.Errorf("not a service account access key")
}
if s.credentialManager == nil {
return nil, fmt.Errorf("credential manager not available")
}
// Find identity by access key
identity, err := s.credentialManager.GetUserByAccessKey(ctx, accessKey)
if err != nil {
return nil, fmt.Errorf("service account not found for access key: %s", accessKey)
}
if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) {
return nil, fmt.Errorf("not a service account")
}
parts := strings.SplitN(identity.GetName(), ":", 3)
if len(parts) < 3 {
return nil, fmt.Errorf("invalid service account ID format")
}
account := &ServiceAccount{
ID: identity.GetName(),
ParentUser: parts[1],
AccessKeyId: accessKey,
Status: StatusActive,
CreateDate: getCreationDate(identity.GetActions()),
Expiration: getExpiration(identity.GetActions()),
}
if identity.Account != nil {
account.Description = identity.Account.GetDisplayName()
}
for _, action := range identity.GetActions() {
if action == disabledAction {
account.Status = StatusInactive
break
}
}
return account, nil
}
weed/admin/dash/types.go
+47 -1
View File
@@ -78,7 +78,7 @@ type S3Bucket struct {
LastModified time.Time `json:"last_modified"`
Quota int64 `json:"quota"` // Quota in bytes, 0 means no quota
QuotaEnabled bool `json:"quota_enabled"` // Whether quota is enabled
VersioningEnabled bool `json:"versioning_enabled"` // Whether versioning is enabled
VersioningStatus string `json:"versioning_status"` // Versioning status: "" (never enabled), "Enabled", or "Suspended"
ObjectLockEnabled bool `json:"object_lock_enabled"` // Whether object lock is enabled
ObjectLockMode string `json:"object_lock_mode"` // Object lock mode: "GOVERNANCE" or "COMPLIANCE"
ObjectLockDuration int32 `json:"object_lock_duration"` // Default retention duration in days
@@ -552,3 +552,49 @@ type CollectionDetailsData struct {
SortBy string `json:"sort_by"`
SortOrder string `json:"sort_order"`
}
// Service Account management structures
type ServiceAccount struct {
ID string `json:"id"`
ParentUser string `json:"parent_user"`
Description string `json:"description,omitempty"`
AccessKeyId string `json:"access_key_id,omitempty"`
SecretAccessKey string `json:"secret_access_key,omitempty"` // Only returned on creation
Status string `json:"status"`
CreateDate time.Time `json:"create_date"`
Expiration time.Time `json:"expiration,omitempty"`
// ServiceAccountId is used when returning a single ID in some API responses
ServiceAccountId string `json:"service_account_id,omitempty"`
// ServiceAccountIds is used when returning a list of IDs owned by a user
ServiceAccountIds []string `json:"service_account_ids,omitempty"`
}
type ServiceAccountsData struct {
Username string `json:"username"`
ServiceAccounts []ServiceAccount `json:"service_accounts"`
TotalAccounts int `json:"total_accounts"`
ActiveAccounts int `json:"active_accounts"`
AvailableUsers []string `json:"available_users"` // For parent user dropdown
LastUpdated time.Time `json:"last_updated"`
}
type CreateServiceAccountRequest struct {
ParentUser string `json:"parent_user"`
Description string `json:"description,omitempty"`
Expiration string `json:"expiration,omitempty"` // RFC3339 format
}
type UpdateServiceAccountRequest struct {
Status string `json:"status,omitempty"` // Active, Inactive
Description string `json:"description,omitempty"`
Expiration string `json:"expiration,omitempty"`
}
// STS Configuration display types
type STSConfigData struct {
Enabled bool `json:"enabled"`
Issuer string `json:"issuer,omitempty"`
TokenDuration string `json:"token_duration,omitempty"`
Providers []string `json:"providers,omitempty"`
LastUpdated time.Time `json:"last_updated"`
}
weed/admin/dash/user_management.go
+3 -2
View File
@@ -326,9 +326,10 @@ func generateSecretKey() string {
func generateAccountId() string {
// Generate 12-digit account ID
b := make([]byte, 8)
b := make([]byte, 4)
rand.Read(b)
return fmt.Sprintf("%012d", b[0]<<24|b[1]<<16|b[2]<<8|b[3])
val := (uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]))
return fmt.Sprintf("%012d", val)
}
func randomInt(max int) int {
weed/admin/handlers/admin_handlers.go
+90 -56
View File
@@ -5,21 +5,24 @@ import (
"time"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
"github.com/seaweedfs/seaweedfs/weed/stats"
)
// AdminHandlers contains all the HTTP handlers for the admin interface
type AdminHandlers struct {
adminServer *dash.AdminServer
authHandlers *AuthHandlers
clusterHandlers *ClusterHandlers
fileBrowserHandlers *FileBrowserHandlers
userHandlers *UserHandlers
policyHandlers *PolicyHandlers
maintenanceHandlers *MaintenanceHandlers
mqHandlers *MessageQueueHandlers
adminServer *dash.AdminServer
authHandlers *AuthHandlers
clusterHandlers *ClusterHandlers
fileBrowserHandlers *FileBrowserHandlers
userHandlers *UserHandlers
policyHandlers *PolicyHandlers
maintenanceHandlers *MaintenanceHandlers
mqHandlers *MessageQueueHandlers
serviceAccountHandlers *ServiceAccountHandlers
}
// NewAdminHandlers creates a new instance of AdminHandlers
@@ -31,23 +34,28 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers {
policyHandlers := NewPolicyHandlers(adminServer)
maintenanceHandlers := NewMaintenanceHandlers(adminServer)
mqHandlers := NewMessageQueueHandlers(adminServer)
serviceAccountHandlers := NewServiceAccountHandlers(adminServer)
return &AdminHandlers{
adminServer: adminServer,
authHandlers: authHandlers,
clusterHandlers: clusterHandlers,
fileBrowserHandlers: fileBrowserHandlers,
userHandlers: userHandlers,
policyHandlers: policyHandlers,
maintenanceHandlers: maintenanceHandlers,
mqHandlers: mqHandlers,
adminServer: adminServer,
authHandlers: authHandlers,
clusterHandlers: clusterHandlers,
fileBrowserHandlers: fileBrowserHandlers,
userHandlers: userHandlers,
policyHandlers: policyHandlers,
maintenanceHandlers: maintenanceHandlers,
mqHandlers: mqHandlers,
serviceAccountHandlers: serviceAccountHandlers,
}
}
// SetupRoutes configures all the routes for the admin interface
func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, password string) {
func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser, adminPassword, readOnlyUser, readOnlyPassword string) {
// Health check (no auth required)
r.GET("/health", h.HealthCheck)
// Prometheus metrics endpoint (no auth required)
r.GET("/metrics", gin.WrapH(promhttp.HandlerFor(stats.Gather, promhttp.HandlerOpts{})))
// Favicon route (no auth required) - redirect to static version
r.GET("/favicon.ico", func(c *gin.Context) {
c.Redirect(http.StatusMovedPermanently, "/static/favicon.ico")
@@ -56,7 +64,7 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
if authRequired {
// Authentication routes (no auth required)
r.GET("/login", h.authHandlers.ShowLogin)
r.POST("/login", h.authHandlers.HandleLogin(username, password))
r.POST("/login", h.authHandlers.HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword))
r.GET("/logout", h.authHandlers.HandleLogout)
// Protected routes group
@@ -72,6 +80,7 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
protected.GET("/object-store/buckets/:bucket", h.ShowBucketDetails)
protected.GET("/object-store/users", h.userHandlers.ShowObjectStoreUsers)
protected.GET("/object-store/policies", h.policyHandlers.ShowPolicies)
protected.GET("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts)
// File browser routes
protected.GET("/files", h.fileBrowserHandlers.ShowFileBrowser)
@@ -80,12 +89,14 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
protected.GET("/cluster/masters", h.clusterHandlers.ShowClusterMasters)
protected.GET("/cluster/filers", h.clusterHandlers.ShowClusterFilers)
protected.GET("/cluster/volume-servers", h.clusterHandlers.ShowClusterVolumeServers)
protected.GET("/cluster/volumes", h.clusterHandlers.ShowClusterVolumes)
protected.GET("/cluster/volumes/:id/:server", h.clusterHandlers.ShowVolumeDetails)
protected.GET("/cluster/collections", h.clusterHandlers.ShowClusterCollections)
protected.GET("/cluster/collections/:name", h.clusterHandlers.ShowCollectionDetails)
protected.GET("/cluster/ec-shards", h.clusterHandlers.ShowClusterEcShards)
protected.GET("/cluster/ec-volumes/:id", h.clusterHandlers.ShowEcVolumeDetails)
// Storage management routes
protected.GET("/storage/volumes", h.clusterHandlers.ShowClusterVolumes)
protected.GET("/storage/volumes/:id/:server", h.clusterHandlers.ShowVolumeDetails)
protected.GET("/storage/collections", h.clusterHandlers.ShowClusterCollections)
protected.GET("/storage/collections/:name", h.clusterHandlers.ShowCollectionDetails)
protected.GET("/storage/ec-shards", h.clusterHandlers.ShowClusterEcShards)
protected.GET("/storage/ec-volumes/:id", h.clusterHandlers.ShowEcVolumeDetails)
// Message Queue management routes
protected.GET("/mq/brokers", h.mqHandlers.ShowBrokers)
@@ -96,9 +107,9 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
protected.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue)
protected.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers)
protected.GET("/maintenance/config", h.maintenanceHandlers.ShowMaintenanceConfig)
protected.POST("/maintenance/config", h.maintenanceHandlers.UpdateMaintenanceConfig)
protected.POST("/maintenance/config", dash.RequireWriteAccess(), h.maintenanceHandlers.UpdateMaintenanceConfig)
protected.GET("/maintenance/config/:taskType", h.maintenanceHandlers.ShowTaskConfig)
protected.POST("/maintenance/config/:taskType", h.maintenanceHandlers.UpdateTaskConfig)
protected.POST("/maintenance/config/:taskType", dash.RequireWriteAccess(), h.maintenanceHandlers.UpdateTaskConfig)
protected.GET("/maintenance/tasks/:id", h.maintenanceHandlers.ShowTaskDetail)
// API routes for AJAX calls
@@ -115,45 +126,55 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
s3Api := api.Group("/s3")
{
s3Api.GET("/buckets", h.adminServer.ListBucketsAPI)
s3Api.POST("/buckets", h.adminServer.CreateBucket)
s3Api.DELETE("/buckets/:bucket", h.adminServer.DeleteBucket)
s3Api.POST("/buckets", dash.RequireWriteAccess(), h.adminServer.CreateBucket)
s3Api.DELETE("/buckets/:bucket", dash.RequireWriteAccess(), h.adminServer.DeleteBucket)
s3Api.GET("/buckets/:bucket", h.adminServer.ShowBucketDetails)
s3Api.PUT("/buckets/:bucket/quota", h.adminServer.UpdateBucketQuota)
s3Api.PUT("/buckets/:bucket/owner", h.adminServer.UpdateBucketOwner)
s3Api.PUT("/buckets/:bucket/quota", dash.RequireWriteAccess(), h.adminServer.UpdateBucketQuota)
s3Api.PUT("/buckets/:bucket/owner", dash.RequireWriteAccess(), h.adminServer.UpdateBucketOwner)
}
// User management API routes
usersApi := api.Group("/users")
{
usersApi.GET("", h.userHandlers.GetUsers)
usersApi.POST("", h.userHandlers.CreateUser)
usersApi.POST("", dash.RequireWriteAccess(), h.userHandlers.CreateUser)
usersApi.GET("/:username", h.userHandlers.GetUserDetails)
usersApi.PUT("/:username", h.userHandlers.UpdateUser)
usersApi.DELETE("/:username", h.userHandlers.DeleteUser)
usersApi.POST("/:username/access-keys", h.userHandlers.CreateAccessKey)
usersApi.DELETE("/:username/access-keys/:accessKeyId", h.userHandlers.DeleteAccessKey)
usersApi.PUT("/:username", dash.RequireWriteAccess(), h.userHandlers.UpdateUser)
usersApi.DELETE("/:username", dash.RequireWriteAccess(), h.userHandlers.DeleteUser)
usersApi.POST("/:username/access-keys", dash.RequireWriteAccess(), h.userHandlers.CreateAccessKey)
usersApi.DELETE("/:username/access-keys/:accessKeyId", dash.RequireWriteAccess(), h.userHandlers.DeleteAccessKey)
usersApi.GET("/:username/policies", h.userHandlers.GetUserPolicies)
usersApi.PUT("/:username/policies", h.userHandlers.UpdateUserPolicies)
usersApi.PUT("/:username/policies", dash.RequireWriteAccess(), h.userHandlers.UpdateUserPolicies)
}
// Service Account management API routes
saApi := api.Group("/service-accounts")
{
saApi.GET("", h.serviceAccountHandlers.GetServiceAccounts)
saApi.POST("", dash.RequireWriteAccess(), h.serviceAccountHandlers.CreateServiceAccount)
saApi.GET("/:id", h.serviceAccountHandlers.GetServiceAccountDetails)
saApi.PUT("/:id", dash.RequireWriteAccess(), h.serviceAccountHandlers.UpdateServiceAccount)
saApi.DELETE("/:id", dash.RequireWriteAccess(), h.serviceAccountHandlers.DeleteServiceAccount)
}
// Object Store Policy management API routes
objectStorePoliciesApi := api.Group("/object-store/policies")
{
objectStorePoliciesApi.GET("", h.policyHandlers.GetPolicies)
objectStorePoliciesApi.POST("", h.policyHandlers.CreatePolicy)
objectStorePoliciesApi.POST("", dash.RequireWriteAccess(), h.policyHandlers.CreatePolicy)
objectStorePoliciesApi.GET("/:name", h.policyHandlers.GetPolicy)
objectStorePoliciesApi.PUT("/:name", h.policyHandlers.UpdatePolicy)
objectStorePoliciesApi.DELETE("/:name", h.policyHandlers.DeletePolicy)
objectStorePoliciesApi.PUT("/:name", dash.RequireWriteAccess(), h.policyHandlers.UpdatePolicy)
objectStorePoliciesApi.DELETE("/:name", dash.RequireWriteAccess(), h.policyHandlers.DeletePolicy)
objectStorePoliciesApi.POST("/validate", h.policyHandlers.ValidatePolicy)
}
// File management API routes
filesApi := api.Group("/files")
{
filesApi.DELETE("/delete", h.fileBrowserHandlers.DeleteFile)
filesApi.DELETE("/delete-multiple", h.fileBrowserHandlers.DeleteMultipleFiles)
filesApi.POST("/create-folder", h.fileBrowserHandlers.CreateFolder)
filesApi.POST("/upload", h.fileBrowserHandlers.UploadFile)
filesApi.DELETE("/delete", dash.RequireWriteAccess(), h.fileBrowserHandlers.DeleteFile)
filesApi.DELETE("/delete-multiple", dash.RequireWriteAccess(), h.fileBrowserHandlers.DeleteMultipleFiles)
filesApi.POST("/create-folder", dash.RequireWriteAccess(), h.fileBrowserHandlers.CreateFolder)
filesApi.POST("/upload", dash.RequireWriteAccess(), h.fileBrowserHandlers.UploadFile)
filesApi.GET("/download", h.fileBrowserHandlers.DownloadFile)
filesApi.GET("/view", h.fileBrowserHandlers.ViewFile)
filesApi.GET("/properties", h.fileBrowserHandlers.GetFileProperties)
@@ -162,32 +183,32 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
// Volume management API routes
volumeApi := api.Group("/volumes")
{
volumeApi.POST("/:id/:server/vacuum", h.clusterHandlers.VacuumVolume)
volumeApi.POST("/:id/:server/vacuum", dash.RequireWriteAccess(), h.clusterHandlers.VacuumVolume)
}
// Maintenance API routes
maintenanceApi := api.Group("/maintenance")
{
maintenanceApi.POST("/scan", h.adminServer.TriggerMaintenanceScan)
maintenanceApi.POST("/scan", dash.RequireWriteAccess(), h.adminServer.TriggerMaintenanceScan)
maintenanceApi.GET("/tasks", h.adminServer.GetMaintenanceTasks)
maintenanceApi.GET("/tasks/:id", h.adminServer.GetMaintenanceTask)
maintenanceApi.GET("/tasks/:id/detail", h.adminServer.GetMaintenanceTaskDetailAPI)
maintenanceApi.POST("/tasks/:id/cancel", h.adminServer.CancelMaintenanceTask)
maintenanceApi.POST("/tasks/:id/cancel", dash.RequireWriteAccess(), h.adminServer.CancelMaintenanceTask)
maintenanceApi.GET("/workers", h.adminServer.GetMaintenanceWorkersAPI)
maintenanceApi.GET("/workers/:id", h.adminServer.GetMaintenanceWorker)
maintenanceApi.GET("/workers/:id/logs", h.adminServer.GetWorkerLogs)
maintenanceApi.GET("/stats", h.adminServer.GetMaintenanceStats)
maintenanceApi.GET("/config", h.adminServer.GetMaintenanceConfigAPI)
maintenanceApi.PUT("/config", h.adminServer.UpdateMaintenanceConfigAPI)
maintenanceApi.PUT("/config", dash.RequireWriteAccess(), h.adminServer.UpdateMaintenanceConfigAPI)
}
// Message Queue API routes
mqApi := api.Group("/mq")
{
mqApi.GET("/topics/:namespace/:topic", h.mqHandlers.GetTopicDetailsAPI)
mqApi.POST("/topics/create", h.mqHandlers.CreateTopicAPI)
mqApi.POST("/topics/retention/update", h.mqHandlers.UpdateTopicRetentionAPI)
mqApi.POST("/retention/purge", h.adminServer.TriggerTopicRetentionPurgeAPI)
mqApi.POST("/topics/create", dash.RequireWriteAccess(), h.mqHandlers.CreateTopicAPI)
mqApi.POST("/topics/retention/update", dash.RequireWriteAccess(), h.mqHandlers.UpdateTopicRetentionAPI)
mqApi.POST("/retention/purge", dash.RequireWriteAccess(), h.adminServer.TriggerTopicRetentionPurgeAPI)
}
}
} else {
@@ -200,6 +221,7 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
r.GET("/object-store/buckets/:bucket", h.ShowBucketDetails)
r.GET("/object-store/users", h.userHandlers.ShowObjectStoreUsers)
r.GET("/object-store/policies", h.policyHandlers.ShowPolicies)
r.GET("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts)
// File browser routes
r.GET("/files", h.fileBrowserHandlers.ShowFileBrowser)
@@ -208,12 +230,14 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
r.GET("/cluster/masters", h.clusterHandlers.ShowClusterMasters)
r.GET("/cluster/filers", h.clusterHandlers.ShowClusterFilers)
r.GET("/cluster/volume-servers", h.clusterHandlers.ShowClusterVolumeServers)
r.GET("/cluster/volumes", h.clusterHandlers.ShowClusterVolumes)
r.GET("/cluster/volumes/:id/:server", h.clusterHandlers.ShowVolumeDetails)
r.GET("/cluster/collections", h.clusterHandlers.ShowClusterCollections)
r.GET("/cluster/collections/:name", h.clusterHandlers.ShowCollectionDetails)
r.GET("/cluster/ec-shards", h.clusterHandlers.ShowClusterEcShards)
r.GET("/cluster/ec-volumes/:id", h.clusterHandlers.ShowEcVolumeDetails)
// Storage management routes
r.GET("/storage/volumes", h.clusterHandlers.ShowClusterVolumes)
r.GET("/storage/volumes/:id/:server", h.clusterHandlers.ShowVolumeDetails)
r.GET("/storage/collections", h.clusterHandlers.ShowClusterCollections)
r.GET("/storage/collections/:name", h.clusterHandlers.ShowCollectionDetails)
r.GET("/storage/ec-shards", h.clusterHandlers.ShowClusterEcShards)
r.GET("/storage/ec-volumes/:id", h.clusterHandlers.ShowEcVolumeDetails)
// Message Queue management routes
r.GET("/mq/brokers", h.mqHandlers.ShowBrokers)
@@ -263,6 +287,16 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username,
usersApi.PUT("/:username/policies", h.userHandlers.UpdateUserPolicies)
}
// Service Account management API routes
saApi := api.Group("/service-accounts")
{
saApi.GET("", h.serviceAccountHandlers.GetServiceAccounts)
saApi.POST("", h.serviceAccountHandlers.CreateServiceAccount)
saApi.GET("/:id", h.serviceAccountHandlers.GetServiceAccountDetails)
saApi.PUT("/:id", h.serviceAccountHandlers.UpdateServiceAccount)
saApi.DELETE("/:id", h.serviceAccountHandlers.DeleteServiceAccount)
}
// Object Store Policy management API routes
objectStorePoliciesApi := api.Group("/object-store/policies")
{
weed/admin/handlers/auth_handlers.go
+11 -2
View File
@@ -3,6 +3,7 @@ package handlers
import (
"net/http"
"github.com/gin-contrib/sessions"
"github.com/gin-gonic/gin"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
@@ -22,6 +23,14 @@ func NewAuthHandlers(adminServer *dash.AdminServer) *AuthHandlers {
// ShowLogin displays the login page
func (a *AuthHandlers) ShowLogin(c *gin.Context) {
session := sessions.Default(c)
// If already authenticated, redirect to admin
if session.Get("authenticated") == true {
c.Redirect(http.StatusSeeOther, "/admin")
return
}
errorMessage := c.Query("error")
// Render login template
@@ -35,8 +44,8 @@ func (a *AuthHandlers) ShowLogin(c *gin.Context) {
}
// HandleLogin handles login form submission
func (a *AuthHandlers) HandleLogin(username, password string) gin.HandlerFunc {
return a.adminServer.HandleLogin(username, password)
func (a *AuthHandlers) HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword string) gin.HandlerFunc {
return a.adminServer.HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword)
}
// HandleLogout handles user logout
weed/admin/handlers/file_browser_handlers.go
+15 -2
View File
@@ -60,9 +60,22 @@ func (h *FileBrowserHandlers) newClientWithTimeout(timeout time.Duration) http.C
func (h *FileBrowserHandlers) ShowFileBrowser(c *gin.Context) {
// Get path from query parameter, default to root
path := c.DefaultQuery("path", "/")
// Normalize Windows-style paths for consistency
path = util.CleanWindowsPath(path)
// Get file browser data
browserData, err := h.adminServer.GetFileBrowser(path)
// Get pagination parameters
lastFileName := c.DefaultQuery("lastFileName", "")
pageSize, err := strconv.Atoi(c.DefaultQuery("limit", "20"))
if err != nil || pageSize < 1 {
pageSize = 20
}
if pageSize > 200 {
pageSize = 200
}
// Get file browser data with cursor-based pagination
browserData, err := h.adminServer.GetFileBrowser(path, lastFileName, pageSize)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get file browser data: " + err.Error()})
return
weed/admin/handlers/service_account_handlers.go
+213
View File
@@ -0,0 +1,213 @@
package handlers
import (
"bytes"
"errors"
"net/http"
"time"
"github.com/gin-gonic/gin"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
// ServiceAccountHandlers contains HTTP handlers for service account management
type ServiceAccountHandlers struct {
adminServer *dash.AdminServer
}
// NewServiceAccountHandlers creates a new instance of ServiceAccountHandlers
func NewServiceAccountHandlers(adminServer *dash.AdminServer) *ServiceAccountHandlers {
return &ServiceAccountHandlers{
adminServer: adminServer,
}
}
// ShowServiceAccounts renders the service accounts management page
func (h *ServiceAccountHandlers) ShowServiceAccounts(c *gin.Context) {
data := h.getServiceAccountsData(c)
// Render to buffer first to avoid partial writes on error
var buf bytes.Buffer
component := app.ServiceAccounts(data)
layoutComponent := layout.Layout(c, component)
err := layoutComponent.Render(c.Request.Context(), &buf)
if err != nil {
glog.Errorf("Failed to render service accounts template: %v", err)
c.AbortWithStatus(http.StatusInternalServerError)
return
}
// Only write to response if render succeeded
c.Header("Content-Type", "text/html")
c.Writer.Write(buf.Bytes())
}
// GetServiceAccounts returns the list of service accounts as JSON
func (h *ServiceAccountHandlers) GetServiceAccounts(c *gin.Context) {
parentUser := c.Query("parent_user")
accounts, err := h.adminServer.GetServiceAccounts(c.Request.Context(), parentUser)
if err != nil {
glog.Errorf("Failed to get service accounts: %v", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get service accounts"})
return
}
c.JSON(http.StatusOK, gin.H{"service_accounts": accounts})
}
// CreateServiceAccount handles service account creation
func (h *ServiceAccountHandlers) CreateServiceAccount(c *gin.Context) {
var req dash.CreateServiceAccountRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()})
return
}
if req.ParentUser == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "ParentUser is required"})
return
}
sa, err := h.adminServer.CreateServiceAccount(c.Request.Context(), req)
if err != nil {
glog.Errorf("Failed to create service account for user %s: %v", req.ParentUser, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create service account"})
return
}
c.JSON(http.StatusCreated, gin.H{
"message": "Service account created successfully",
"service_account": sa,
})
}
// GetServiceAccountDetails returns detailed information about a service account
func (h *ServiceAccountHandlers) GetServiceAccountDetails(c *gin.Context) {
id := c.Param("id")
if id == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "Service account ID is required"})
return
}
sa, err := h.adminServer.GetServiceAccountDetails(c.Request.Context(), id)
if err != nil {
// Distinguish not-found errors from internal errors
if errors.Is(err, dash.ErrServiceAccountNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "Service account not found: " + err.Error()})
} else {
glog.Errorf("Failed to get service account details for %s: %v", id, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get service account details"})
}
return
}
c.JSON(http.StatusOK, sa)
}
// UpdateServiceAccount handles service account updates
func (h *ServiceAccountHandlers) UpdateServiceAccount(c *gin.Context) {
id := c.Param("id")
if id == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "Service account ID is required"})
return
}
var req dash.UpdateServiceAccountRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()})
return
}
sa, err := h.adminServer.UpdateServiceAccount(c.Request.Context(), id, req)
if err != nil {
// Distinguish not-found errors from internal errors
if errors.Is(err, dash.ErrServiceAccountNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "Service account not found"})
} else {
glog.Errorf("Failed to update service account %s: %v", id, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update service account"})
}
return
}
c.JSON(http.StatusOK, gin.H{
"message": "Service account updated successfully",
"service_account": sa,
})
}
// DeleteServiceAccount handles service account deletion
func (h *ServiceAccountHandlers) DeleteServiceAccount(c *gin.Context) {
id := c.Param("id")
if id == "" {
c.JSON(http.StatusBadRequest, gin.H{"error": "Service account ID is required"})
return
}
err := h.adminServer.DeleteServiceAccount(c.Request.Context(), id)
if err != nil {
// Distinguish not-found errors from internal errors
if errors.Is(err, dash.ErrServiceAccountNotFound) {
c.JSON(http.StatusNotFound, gin.H{"error": "Service account not found"})
} else {
glog.Errorf("Failed to delete service account %s: %v", id, err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete service account"})
}
return
}
c.JSON(http.StatusOK, gin.H{
"message": "Service account deleted successfully",
})
}
// getServiceAccountsData retrieves service accounts data for the template
func (h *ServiceAccountHandlers) getServiceAccountsData(c *gin.Context) dash.ServiceAccountsData {
username := c.GetString("username")
if username == "" {
username = "admin"
}
// Get all service accounts
accounts, err := h.adminServer.GetServiceAccounts(c.Request.Context(), "")
if err != nil {
glog.Errorf("Failed to get service accounts: %v", err)
return dash.ServiceAccountsData{
Username: username,
ServiceAccounts: []dash.ServiceAccount{},
TotalAccounts: 0,
LastUpdated: time.Now(),
}
}
// Count active accounts
activeCount := 0
for _, sa := range accounts {
if sa.Status == dash.StatusActive {
activeCount++
}
}
// Get available users for dropdown
var availableUsers []string
users, err := h.adminServer.GetObjectStoreUsers(c.Request.Context())
if err != nil {
glog.Errorf("Failed to get users for dropdown: %v", err)
} else {
for _, user := range users {
availableUsers = append(availableUsers, user.Username)
}
}
return dash.ServiceAccountsData{
Username: username,
ServiceAccounts: accounts,
TotalAccounts: len(accounts),
ActiveAccounts: activeCount,
AvailableUsers: availableUsers,
LastUpdated: time.Now(),
}
}
weed/admin/handlers/user_handlers.go
+2 -2
View File
@@ -41,7 +41,7 @@ func (h *UserHandlers) ShowObjectStoreUsers(c *gin.Context) {
// GetUsers returns the list of users as JSON
func (h *UserHandlers) GetUsers(c *gin.Context) {
users, err := h.adminServer.GetObjectStoreUsers()
users, err := h.adminServer.GetObjectStoreUsers(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get users: " + err.Error()})
return
@@ -234,7 +234,7 @@ func (h *UserHandlers) getObjectStoreUsersData(c *gin.Context) dash.ObjectStoreU
}
// Get object store users
users, err := h.adminServer.GetObjectStoreUsers()
users, err := h.adminServer.GetObjectStoreUsers(c.Request.Context())
if err != nil {
glog.Errorf("Failed to get object store users: %v", err)
// Return empty data on error
weed/admin/maintenance/maintenance_scanner.go
+2
View File
@@ -115,6 +115,7 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics,
metric := &VolumeHealthMetrics{
VolumeID: volInfo.Id,
Server: node.Id,
ServerAddress: node.Address,
DiskType: diskType, // Track which disk this volume is on
DiskId: volInfo.DiskId, // Use disk ID from volume info
DataCenter: dc.Id, // Data center from current loop
@@ -207,6 +208,7 @@ func (ms *MaintenanceScanner) convertToTaskMetrics(metrics []*VolumeHealthMetric
simplified = append(simplified, &types.VolumeHealthMetrics{
VolumeID: metric.VolumeID,
Server: metric.Server,
ServerAddress: metric.ServerAddress,
DiskType: metric.DiskType,
DiskId: metric.DiskId,
DataCenter: metric.DataCenter,
weed/admin/maintenance/maintenance_types.go
+1
View File
@@ -362,6 +362,7 @@ type TaskDetectionResult struct {
type VolumeHealthMetrics struct {
VolumeID uint32 `json:"volume_id"`
Server string `json:"server"`
ServerAddress string `json:"server_address"`
DiskType string `json:"disk_type"` // Disk type (e.g., "hdd", "ssd") or disk path (e.g., "/data1")
DiskId uint32 `json:"disk_id"` // ID of the disk in Store.Locations array
DataCenter string `json:"data_center"` // Data center of the server
weed/admin/maintenance/pending_operations_test.go
+4 -4
View File
@@ -110,10 +110,10 @@ func TestPendingOperations_VolumeFiltering(t *testing.T) {
// Create volume metrics
metrics := []*types.VolumeHealthMetrics{
{VolumeID: 100, Server: "node1"},
{VolumeID: 101, Server: "node2"},
{VolumeID: 102, Server: "node3"},
{VolumeID: 103, Server: "node1"},
{VolumeID: 100, Server: "node1", ServerAddress: "192.168.1.1:8080"},
{VolumeID: 101, Server: "node2", ServerAddress: "192.168.1.2:8080"},
{VolumeID: 102, Server: "node3", ServerAddress: "192.168.1.3:8080"},
{VolumeID: 103, Server: "node1", ServerAddress: "192.168.1.1:8080"},
}
// Add pending operations on volumes 101 and 103
weed/admin/topology/structs.go
+1
View File
@@ -97,6 +97,7 @@ type ActiveTopology struct {
// DestinationPlan represents a planned destination for a volume/shard operation
type DestinationPlan struct {
TargetNode string `json:"target_node"`
TargetAddress string `json:"target_address"`
TargetDisk uint32 `json:"target_disk"`
TargetRack string `json:"target_rack"`
TargetDC string `json:"target_dc"`
weed/admin/view/app/cluster_collections.templ
+5 -5
View File
@@ -149,12 +149,12 @@ templ ClusterCollections(data dash.ClusterCollectionsData) {
for _, collection := range data.Collections {
<tr>
<td>
<a href={templ.SafeURL(fmt.Sprintf("/cluster/collections/%s", collection.Name))} class="text-decoration-none">
<a href={templ.SafeURL(fmt.Sprintf("/storage/collections/%s", collection.Name))} class="text-decoration-none">
<strong>{collection.Name}</strong>
</a>
</td>
<td>
<a href={templ.SafeURL(fmt.Sprintf("/cluster/volumes?collection=%s", collection.Name))} class="text-decoration-none">
<a href={templ.SafeURL(fmt.Sprintf("/storage/volumes?collection=%s", collection.Name))} class="text-decoration-none">
<div class="d-flex align-items-center">
<i class="fas fa-database me-2 text-muted"></i>
if collection.VolumeCount > 0 {
@@ -166,7 +166,7 @@ templ ClusterCollections(data dash.ClusterCollectionsData) {
</a>
</td>
<td>
<a href={templ.SafeURL(fmt.Sprintf("/cluster/ec-shards?collection=%s", collection.Name))} class="text-decoration-none">
<a href={templ.SafeURL(fmt.Sprintf("/storage/ec-shards?collection=%s", collection.Name))} class="text-decoration-none">
<div class="d-flex align-items-center">
<i class="fas fa-th-large me-2 text-muted"></i>
if collection.EcVolumeCount > 0 {
@@ -330,10 +330,10 @@ templ ClusterCollections(data dash.ClusterCollectionsData) {
'<div class="col-12">' +
'<h6 class="text-primary"><i class="fas fa-link me-1"></i>Quick Actions</h6>' +
'<div class="d-grid gap-2 d-md-flex">' +
'<a href="/cluster/volumes?collection=' + encodeURIComponent(collection.name) + '" class="btn btn-outline-primary">' +
'<a href="/storage/volumes?collection=' + encodeURIComponent(collection.name) + '" class="btn btn-outline-primary">' +
'<i class="fas fa-database me-1"></i>View Volumes' +
'</a>' +
'<a href="/cluster/ec-shards?collection=' + encodeURIComponent(collection.name) + '" class="btn btn-outline-secondary">' +
'<a href="/storage/ec-shards?collection=' + encodeURIComponent(collection.name) + '" class="btn btn-outline-secondary">' +
'<i class="fas fa-th-large me-1"></i>View EC Volumes' +
'</a>' +
'<a href="/files?collection=' + encodeURIComponent(collection.name) + '" class="btn btn-outline-info">' +
weed/admin/view/app/cluster_collections_templ.go
+4 -4
View File
File diff suppressed because one or more lines are too long
weed/admin/view/app/cluster_ec_shards.templ
+6 -6
View File
@@ -22,7 +22,7 @@ templ ClusterEcShards(data dash.ClusterEcShardsData) {
<i class="fas fa-filter me-1"></i>Collection: {data.FilterCollection}
</span>
}
<a href="/cluster/ec-shards" class="btn btn-sm btn-outline-secondary">
<a href="/storage/ec-shards" class="btn btn-sm btn-outline-secondary">
<i class="fas fa-times me-1"></i>Clear Filter
</a>
</div>
@@ -205,11 +205,11 @@ templ ClusterEcShards(data dash.ClusterEcShardsData) {
if data.ShowCollectionColumn {
<td>
if shard.Collection != "" {
<a href="/cluster/ec-shards?collection={shard.Collection}" class="text-decoration-none">
<a href="/storage/ec-shards?collection={shard.Collection}" class="text-decoration-none">
<span class="badge bg-info text-white">{shard.Collection}</span>
</a>
} else {
<a href="/cluster/ec-shards?collection=default" class="text-decoration-none">
<a href="/storage/ec-shards?collection=default" class="text-decoration-none">
<span class="badge bg-secondary text-white">default</span>
</a>
}
@@ -366,7 +366,7 @@ templ ClusterEcShards(data dash.ClusterEcShardsData) {
}
function exportEcShards() {
const url = new URL('/api/cluster/ec-shards/export', window.location.origin);
const url = new URL('/api/storage/ec-shards/export', window.location.origin);
const params = new URLSearchParams(window.location.search);
params.forEach((value, key) => {
url.searchParams.set(key, value);
@@ -380,7 +380,7 @@ templ ClusterEcShards(data dash.ClusterEcShardsData) {
const volumeId = button.getAttribute('data-volume-id');
// Navigate to the EC volume details page
window.location.href = `/cluster/ec-volumes/${volumeId}`;
window.location.href = `/storage/ec-volumes/${volumeId}`;
}
function repairVolume(event) {
@@ -388,7 +388,7 @@ templ ClusterEcShards(data dash.ClusterEcShardsData) {
const button = event.target.closest('button');
const volumeId = button.getAttribute('data-volume-id');
if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) {
fetch(`/api/cluster/volumes/${volumeId}/repair`, {
fetch(`/api/storage/volumes/${volumeId}/repair`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
weed/admin/view/app/cluster_ec_shards_templ.go
+4 -4
View File
@@ -67,7 +67,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<a href=\"/cluster/ec-shards\" class=\"btn btn-sm btn-outline-secondary\"><i class=\"fas fa-times me-1\"></i>Clear Filter</a></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<a href=\"/storage/ec-shards\" class=\"btn btn-sm btn-outline-secondary\"><i class=\"fas fa-times me-1\"></i>Clear Filter</a></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -328,7 +328,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
return templ_7745c5c3_Err
}
if shard.Collection != "" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "<a href=\"/cluster/ec-shards?collection={shard.Collection}\" class=\"text-decoration-none\"><span class=\"badge bg-info text-white\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "<a href=\"/storage/ec-shards?collection={shard.Collection}\" class=\"text-decoration-none\"><span class=\"badge bg-info text-white\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -346,7 +346,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "<a href=\"/cluster/ec-shards?collection=default\" class=\"text-decoration-none\"><span class=\"badge bg-secondary text-white\">default</span></a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "<a href=\"/storage/ec-shards?collection=default\" class=\"text-decoration-none\"><span class=\"badge bg-secondary text-white\">default</span></a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -663,7 +663,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 89, "<!-- JavaScript --><script>\n function sortBy(field) {\n const currentSort = \"{data.SortBy}\";\n const currentOrder = \"{data.SortOrder}\";\n let newOrder = 'asc';\n \n if (currentSort === field && currentOrder === 'asc') {\n newOrder = 'desc';\n }\n \n updateUrl({\n sortBy: field,\n sortOrder: newOrder,\n page: 1\n });\n }\n\n function goToPage(event) {\n // Get data from the link element (not any child elements)\n const link = event.target.closest('a');\n const page = link.getAttribute('data-page');\n updateUrl({ page: page });\n }\n\n function changePageSize() {\n const pageSize = document.getElementById('pageSizeSelect').value;\n updateUrl({ pageSize: pageSize, page: 1 });\n }\n\n function updateUrl(params) {\n const url = new URL(window.location);\n Object.keys(params).forEach(key => {\n if (params[key]) {\n url.searchParams.set(key, params[key]);\n } else {\n url.searchParams.delete(key);\n }\n });\n window.location.href = url.toString();\n }\n\n function exportEcShards() {\n const url = new URL('/api/cluster/ec-shards/export', window.location.origin);\n const params = new URLSearchParams(window.location.search);\n params.forEach((value, key) => {\n url.searchParams.set(key, value);\n });\n window.open(url.toString(), '_blank');\n }\n\n function showShardDetails(event) {\n // Get data from the button element (not the icon inside it)\n const button = event.target.closest('button');\n const volumeId = button.getAttribute('data-volume-id');\n \n // Navigate to the EC volume details page\n window.location.href = `/cluster/ec-volumes/${volumeId}`;\n }\n\n function repairVolume(event) {\n // Get data from the button element (not the icon inside it)\n const button = event.target.closest('button');\n const volumeId = button.getAttribute('data-volume-id');\n if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) {\n fetch(`/api/cluster/volumes/${volumeId}/repair`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n }\n })\n .then(response => response.json())\n .then(data => {\n if (data.success) {\n alert('Repair initiated successfully');\n location.reload();\n } else {\n alert('Failed to initiate repair: ' + data.error);\n }\n })\n .catch(error => {\n alert('Error: ' + error.message);\n });\n }\n }\n </script>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 89, "<!-- JavaScript --><script>\n function sortBy(field) {\n const currentSort = \"{data.SortBy}\";\n const currentOrder = \"{data.SortOrder}\";\n let newOrder = 'asc';\n \n if (currentSort === field && currentOrder === 'asc') {\n newOrder = 'desc';\n }\n \n updateUrl({\n sortBy: field,\n sortOrder: newOrder,\n page: 1\n });\n }\n\n function goToPage(event) {\n // Get data from the link element (not any child elements)\n const link = event.target.closest('a');\n const page = link.getAttribute('data-page');\n updateUrl({ page: page });\n }\n\n function changePageSize() {\n const pageSize = document.getElementById('pageSizeSelect').value;\n updateUrl({ pageSize: pageSize, page: 1 });\n }\n\n function updateUrl(params) {\n const url = new URL(window.location);\n Object.keys(params).forEach(key => {\n if (params[key]) {\n url.searchParams.set(key, params[key]);\n } else {\n url.searchParams.delete(key);\n }\n });\n window.location.href = url.toString();\n }\n\n function exportEcShards() {\n const url = new URL('/api/storage/ec-shards/export', window.location.origin);\n const params = new URLSearchParams(window.location.search);\n params.forEach((value, key) => {\n url.searchParams.set(key, value);\n });\n window.open(url.toString(), '_blank');\n }\n\n function showShardDetails(event) {\n // Get data from the button element (not the icon inside it)\n const button = event.target.closest('button');\n const volumeId = button.getAttribute('data-volume-id');\n \n // Navigate to the EC volume details page\n window.location.href = `/storage/ec-volumes/${volumeId}`;\n }\n\n function repairVolume(event) {\n // Get data from the button element (not the icon inside it)\n const button = event.target.closest('button');\n const volumeId = button.getAttribute('data-volume-id');\n if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) {\n fetch(`/api/storage/volumes/${volumeId}/repair`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n }\n })\n .then(response => response.json())\n .then(data => {\n if (data.success) {\n alert('Repair initiated successfully');\n location.reload();\n } else {\n alert('Failed to initiate repair: ' + data.error);\n }\n })\n .catch(error => {\n alert('Error: ' + error.message);\n });\n }\n }\n </script>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
weed/admin/view/app/cluster_ec_volumes.templ
+5 -5
View File
@@ -25,7 +25,7 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
<i class="fas fa-filter me-1"></i>Collection: {data.Collection}
</span>
}
<a href="/cluster/ec-volumes" class="btn btn-sm btn-outline-secondary">
<a href="/storage/ec-volumes" class="btn btn-sm btn-outline-secondary">
<i class="fas fa-times me-1"></i>Clear Filter
</a>
</div>
@@ -201,11 +201,11 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
if data.ShowCollectionColumn {
<td>
if volume.Collection != "" {
<a href={ templ.URL(fmt.Sprintf("/cluster/ec-shards?collection=%s", volume.Collection)) } class="text-decoration-none">
<a href={ templ.URL(fmt.Sprintf("/storage/ec-shards?collection=%s", volume.Collection)) } class="text-decoration-none">
<span class="badge bg-info text-white">{volume.Collection}</span>
</a>
} else {
<a href={ templ.URL("/cluster/ec-shards?collection=default") } class="text-decoration-none">
<a href={ templ.URL("/storage/ec-shards?collection=default") } class="text-decoration-none">
<span class="badge bg-secondary text-white">default</span>
</a>
}
@@ -373,13 +373,13 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) {
function showVolumeDetails(event) {
const volumeId = event.target.closest('button').getAttribute('data-volume-id');
window.location.href = `/cluster/ec-volumes/${volumeId}`;
window.location.href = `/storage/ec-volumes/${volumeId}`;
}
function repairVolume(event) {
const volumeId = event.target.closest('button').getAttribute('data-volume-id');
if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) {
fetch(`/api/cluster/ec-volumes/${volumeId}/repair`, {
fetch(`/api/storage/ec-volumes/${volumeId}/repair`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
weed/admin/view/app/cluster_ec_volumes_templ.go
+4 -4
View File
@@ -70,7 +70,7 @@ func ClusterEcVolumes(data dash.ClusterEcVolumesData) templ.Component {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<a href=\"/cluster/ec-volumes\" class=\"btn btn-sm btn-outline-secondary\"><i class=\"fas fa-times me-1\"></i>Clear Filter</a></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<a href=\"/storage/ec-volumes\" class=\"btn btn-sm btn-outline-secondary\"><i class=\"fas fa-times me-1\"></i>Clear Filter</a></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -383,7 +383,7 @@ func ClusterEcVolumes(data dash.ClusterEcVolumesData) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var13 templ.SafeURL
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinURLErrs(templ.URL(fmt.Sprintf("/cluster/ec-shards?collection=%s", volume.Collection)))
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinURLErrs(templ.URL(fmt.Sprintf("/storage/ec-shards?collection=%s", volume.Collection)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_volumes.templ`, Line: 204, Col: 123}
}
@@ -414,7 +414,7 @@ func ClusterEcVolumes(data dash.ClusterEcVolumesData) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var15 templ.SafeURL
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinURLErrs(templ.URL("/cluster/ec-shards?collection=default"))
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinURLErrs(templ.URL("/storage/ec-shards?collection=default"))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_volumes.templ`, Line: 208, Col: 96}
}
@@ -757,7 +757,7 @@ func ClusterEcVolumes(data dash.ClusterEcVolumesData) templ.Component {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "<!-- JavaScript --><script>\n function sortBy(field) {\n const currentSort = new URLSearchParams(window.location.search).get('sort_by');\n const currentOrder = new URLSearchParams(window.location.search).get('sort_order') || 'asc';\n \n let newOrder = 'asc';\n if (currentSort === field && currentOrder === 'asc') {\n newOrder = 'desc';\n }\n \n updateUrl({\n sort_by: field,\n sort_order: newOrder,\n page: 1\n });\n }\n\n function goToPage(event) {\n event.preventDefault();\n const page = event.target.closest('a').getAttribute('data-page');\n updateUrl({ page: page });\n }\n\n function changePageSize(newPageSize) {\n updateUrl({ page_size: newPageSize, page: 1 });\n }\n\n function updateUrl(params) {\n const url = new URL(window.location);\n Object.keys(params).forEach(key => {\n if (params[key] != null) {\n url.searchParams.set(key, params[key]);\n } else {\n url.searchParams.delete(key);\n }\n });\n window.location.href = url.toString();\n }\n\n function showVolumeDetails(event) {\n const volumeId = event.target.closest('button').getAttribute('data-volume-id');\n window.location.href = `/cluster/ec-volumes/${volumeId}`;\n }\n\n function repairVolume(event) {\n const volumeId = event.target.closest('button').getAttribute('data-volume-id');\n if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) {\n fetch(`/api/cluster/ec-volumes/${volumeId}/repair`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n }\n })\n .then(response => {\n if (!response.ok) {\n throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n }\n return response.json();\n })\n .then(data => {\n if (data && data.success) {\n alert('Repair initiated successfully');\n location.reload();\n } else {\n alert('Failed to initiate repair: ' + (data && data.error ? data.error : 'Unknown error'));\n }\n })\n .catch(error => {\n alert('Error: ' + error.message);\n });\n }\n }\n </script>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "<!-- JavaScript --><script>\n function sortBy(field) {\n const currentSort = new URLSearchParams(window.location.search).get('sort_by');\n const currentOrder = new URLSearchParams(window.location.search).get('sort_order') || 'asc';\n \n let newOrder = 'asc';\n if (currentSort === field && currentOrder === 'asc') {\n newOrder = 'desc';\n }\n \n updateUrl({\n sort_by: field,\n sort_order: newOrder,\n page: 1\n });\n }\n\n function goToPage(event) {\n event.preventDefault();\n const page = event.target.closest('a').getAttribute('data-page');\n updateUrl({ page: page });\n }\n\n function changePageSize(newPageSize) {\n updateUrl({ page_size: newPageSize, page: 1 });\n }\n\n function updateUrl(params) {\n const url = new URL(window.location);\n Object.keys(params).forEach(key => {\n if (params[key] != null) {\n url.searchParams.set(key, params[key]);\n } else {\n url.searchParams.delete(key);\n }\n });\n window.location.href = url.toString();\n }\n\n function showVolumeDetails(event) {\n const volumeId = event.target.closest('button').getAttribute('data-volume-id');\n window.location.href = `/storage/ec-volumes/${volumeId}`;\n }\n\n function repairVolume(event) {\n const volumeId = event.target.closest('button').getAttribute('data-volume-id');\n if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) {\n fetch(`/api/storage/ec-volumes/${volumeId}/repair`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n }\n })\n .then(response => {\n if (!response.ok) {\n throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n }\n return response.json();\n })\n .then(data => {\n if (data && data.success) {\n alert('Repair initiated successfully');\n location.reload();\n } else {\n alert('Failed to initiate repair: ' + (data && data.error ? data.error : 'Unknown error'));\n }\n })\n .catch(error => {\n alert('Error: ' + error.message);\n });\n }\n }\n </script>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
weed/admin/view/app/cluster_volume_servers.templ
+1 -1
View File
@@ -337,7 +337,7 @@ templ ClusterVolumeServers(data dash.ClusterVolumeServersData) {
'<a href="http://' + (server.publicUrl || server.address) + '/ui/index.html" target="_blank" class="btn btn-outline-primary">' +
'<i class="fas fa-external-link-alt me-1"></i>Open Volume Server UI' +
'</a>' +
'<a href="/cluster/volumes?server=' + encodeURIComponent(server.address) + '" class="btn btn-outline-info">' +
'<a href="/storage/volumes?server=' + encodeURIComponent(server.address) + '" class="btn btn-outline-info">' +
'<i class="fas fa-database me-1"></i>View Volumes' +
'</a>' +
'</div>' +
weed/admin/view/app/cluster_volume_servers_templ.go
+1 -1
View File
File diff suppressed because one or more lines are too long
weed/admin/view/app/cluster_volumes.templ
+4 -4
View File
@@ -17,7 +17,7 @@ templ ClusterVolumes(data dash.ClusterVolumesData) {
<span class="badge bg-info me-2">
<i class="fas fa-filter me-1"></i>Collection: {data.FilterCollection}
</span>
<a href="/cluster/volumes" class="btn btn-sm btn-outline-secondary">
<a href="/storage/volumes" class="btn btn-sm btn-outline-secondary">
<i class="fas fa-times me-1"></i>Clear Filter
</a>
</div>
@@ -338,11 +338,11 @@ templ ClusterVolumes(data dash.ClusterVolumesData) {
if data.ShowCollectionColumn {
<td>
if volume.Collection == "" {
<a href={templ.SafeURL("/cluster/volumes?collection=default")} class="text-decoration-none">
<a href={templ.SafeURL("/storage/volumes?collection=default")} class="text-decoration-none">
<span class="badge bg-secondary">default</span>
</a>
} else {
<a href={templ.SafeURL(fmt.Sprintf("/cluster/volumes?collection=%s", volume.Collection))} class="text-decoration-none">
<a href={templ.SafeURL(fmt.Sprintf("/storage/volumes?collection=%s", volume.Collection))} class="text-decoration-none">
<span class="badge bg-secondary">{volume.Collection}</span>
</a>
}
@@ -597,7 +597,7 @@ templ ClusterVolumes(data dash.ClusterVolumesData) {
const serverCell = row.querySelector('td:nth-child(2) a');
const server = serverCell ? serverCell.textContent.trim() : 'unknown';
window.location.href = `/cluster/volumes/${volumeId}/${encodeURIComponent(server)}`;
window.location.href = `/storage/volumes/${volumeId}/${encodeURIComponent(server)}`;
}
function performVacuum(volumeId, server, button) {
weed/admin/view/app/cluster_volumes_templ.go
+4 -4
View File
File diff suppressed because one or more lines are too long
weed/admin/view/app/collection_details.templ
+3 -3
View File
@@ -15,7 +15,7 @@ templ CollectionDetails(data dash.CollectionDetailsData) {
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
<li class="breadcrumb-item"><a href="/admin" class="text-decoration-none">Dashboard</a></li>
<li class="breadcrumb-item"><a href="/cluster/collections" class="text-decoration-none">Collections</a></li>
<li class="breadcrumb-item"><a href="/storage/collections" class="text-decoration-none">Collections</a></li>
<li class="breadcrumb-item active" aria-current="page">{data.CollectionName}</li>
</ol>
</nav>
@@ -360,13 +360,13 @@ templ CollectionDetails(data dash.CollectionDetailsData) {
function showVolumeDetails(event) {
const volumeId = event.target.closest('button').getAttribute('data-volume-id');
const server = event.target.closest('button').getAttribute('data-server');
window.location.href = `/cluster/volumes/${volumeId}/${server}`;
window.location.href = `/storage/volumes/${volumeId}/${server}`;
}
// EC Volume details
function showEcVolumeDetails(event) {
const volumeId = event.target.closest('button').getAttribute('data-volume-id');
window.location.href = `/cluster/ec-volumes/${volumeId}`;
window.location.href = `/storage/ec-volumes/${volumeId}`;
}
// Repair EC Volume
weed/admin/view/app/collection_details_templ.go
+2 -2
View File
@@ -48,7 +48,7 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "</h1><nav aria-label=\"breadcrumb\"><ol class=\"breadcrumb\"><li class=\"breadcrumb-item\"><a href=\"/admin\" class=\"text-decoration-none\">Dashboard</a></li><li class=\"breadcrumb-item\"><a href=\"/cluster/collections\" class=\"text-decoration-none\">Collections</a></li><li class=\"breadcrumb-item active\" aria-current=\"page\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "</h1><nav aria-label=\"breadcrumb\"><ol class=\"breadcrumb\"><li class=\"breadcrumb-item\"><a href=\"/admin\" class=\"text-decoration-none\">Dashboard</a></li><li class=\"breadcrumb-item\"><a href=\"/storage/collections\" class=\"text-decoration-none\">Collections</a></li><li class=\"breadcrumb-item active\" aria-current=\"page\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -575,7 +575,7 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "<script>\n\t\t// Sorting functionality\n\t\tfunction sortBy(field) {\n\t\t\tconst currentSort = new URLSearchParams(window.location.search).get('sort_by');\n\t\t\tconst currentOrder = new URLSearchParams(window.location.search).get('sort_order') || 'asc';\n\t\t\t\n\t\t\tlet newOrder = 'asc';\n\t\t\tif (currentSort === field && currentOrder === 'asc') {\n\t\t\t\tnewOrder = 'desc';\n\t\t\t}\n\t\t\t\n\t\t\tconst url = new URL(window.location);\n\t\t\turl.searchParams.set('sort_by', field);\n\t\t\turl.searchParams.set('sort_order', newOrder);\n\t\t\turl.searchParams.set('page', '1'); // Reset to first page\n\t\t\twindow.location.href = url.toString();\n\t\t}\n\n\t\t// Pagination functionality\n\t\tfunction goToPage(event) {\n\t\t\tevent.preventDefault();\n\t\t\tconst page = event.target.closest('a').getAttribute('data-page');\n\t\t\tconst url = new URL(window.location);\n\t\t\turl.searchParams.set('page', page);\n\t\t\twindow.location.href = url.toString();\n\t\t}\n\n\t\t// Page size functionality\n\t\tfunction changePageSize(newPageSize) {\n\t\t\tconst url = new URL(window.location);\n\t\t\turl.searchParams.set('page_size', newPageSize);\n\t\t\turl.searchParams.set('page', '1'); // Reset to first page when changing page size\n\t\t\twindow.location.href = url.toString();\n\t\t}\n\n\t\t// Volume details\n\t\tfunction showVolumeDetails(event) {\n\t\t\tconst volumeId = event.target.closest('button').getAttribute('data-volume-id');\n\t\t\tconst server = event.target.closest('button').getAttribute('data-server');\n\t\t\twindow.location.href = `/cluster/volumes/${volumeId}/${server}`;\n\t\t}\n\n\t\t// EC Volume details\n\t\tfunction showEcVolumeDetails(event) {\n\t\t\tconst volumeId = event.target.closest('button').getAttribute('data-volume-id');\n\t\t\twindow.location.href = `/cluster/ec-volumes/${volumeId}`;\n\t\t}\n\n\t\t// Repair EC Volume\n\t\tfunction repairEcVolume(event) {\n\t\t\tconst volumeId = event.target.closest('button').getAttribute('data-volume-id');\n\t\t\tif (confirm(`Are you sure you want to repair missing shards for EC volume ${volumeId}?`)) {\n\t\t\t\t// TODO: Implement repair functionality\n\t\t\t\talert('Repair functionality will be implemented soon.');\n\t\t\t}\n\t\t}\n\t</script>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "<script>\n\t\t// Sorting functionality\n\t\tfunction sortBy(field) {\n\t\t\tconst currentSort = new URLSearchParams(window.location.search).get('sort_by');\n\t\t\tconst currentOrder = new URLSearchParams(window.location.search).get('sort_order') || 'asc';\n\t\t\t\n\t\t\tlet newOrder = 'asc';\n\t\t\tif (currentSort === field && currentOrder === 'asc') {\n\t\t\t\tnewOrder = 'desc';\n\t\t\t}\n\t\t\t\n\t\t\tconst url = new URL(window.location);\n\t\t\turl.searchParams.set('sort_by', field);\n\t\t\turl.searchParams.set('sort_order', newOrder);\n\t\t\turl.searchParams.set('page', '1'); // Reset to first page\n\t\t\twindow.location.href = url.toString();\n\t\t}\n\n\t\t// Pagination functionality\n\t\tfunction goToPage(event) {\n\t\t\tevent.preventDefault();\n\t\t\tconst page = event.target.closest('a').getAttribute('data-page');\n\t\t\tconst url = new URL(window.location);\n\t\t\turl.searchParams.set('page', page);\n\t\t\twindow.location.href = url.toString();\n\t\t}\n\n\t\t// Page size functionality\n\t\tfunction changePageSize(newPageSize) {\n\t\t\tconst url = new URL(window.location);\n\t\t\turl.searchParams.set('page_size', newPageSize);\n\t\t\turl.searchParams.set('page', '1'); // Reset to first page when changing page size\n\t\t\twindow.location.href = url.toString();\n\t\t}\n\n\t\t// Volume details\n\t\tfunction showVolumeDetails(event) {\n\t\t\tconst volumeId = event.target.closest('button').getAttribute('data-volume-id');\n\t\t\tconst server = event.target.closest('button').getAttribute('data-server');\n\t\t\twindow.location.href = `/storage/volumes/${volumeId}/${server}`;\n\t\t}\n\n\t\t// EC Volume details\n\t\tfunction showEcVolumeDetails(event) {\n\t\t\tconst volumeId = event.target.closest('button').getAttribute('data-volume-id');\n\t\t\twindow.location.href = `/storage/ec-volumes/${volumeId}`;\n\t\t}\n\n\t\t// Repair EC Volume\n\t\tfunction repairEcVolume(event) {\n\t\t\tconst volumeId = event.target.closest('button').getAttribute('data-volume-id');\n\t\t\tif (confirm(`Are you sure you want to repair missing shards for EC volume ${volumeId}?`)) {\n\t\t\t\t// TODO: Implement repair functionality\n\t\t\t\talert('Repair functionality will be implemented soon.');\n\t\t\t}\n\t\t}\n\t</script>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
weed/admin/view/app/ec_volume_details.templ
+1 -1
View File
@@ -14,7 +14,7 @@ templ EcVolumeDetails(data dash.EcVolumeDetailsData) {
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
<li class="breadcrumb-item"><a href="/admin" class="text-decoration-none">Dashboard</a></li>
<li class="breadcrumb-item"><a href="/cluster/ec-shards" class="text-decoration-none">EC Volumes</a></li>
<li class="breadcrumb-item"><a href="/storage/ec-shards" class="text-decoration-none">EC Volumes</a></li>
<li class="breadcrumb-item active" aria-current="page">Volume {fmt.Sprintf("%d", data.VolumeID)}</li>
</ol>
</nav>
weed/admin/view/app/ec_volume_details_templ.go
+1 -1
View File
@@ -34,7 +34,7 @@ func EcVolumeDetails(data dash.EcVolumeDetailsData) templ.Component {
templ_7745c5c3_Var1 = templ.NopComponent
}
ctx = templ.ClearChildren(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><div><h1 class=\"h2\"><i class=\"fas fa-th-large me-2\"></i>EC Volume Details</h1><nav aria-label=\"breadcrumb\"><ol class=\"breadcrumb\"><li class=\"breadcrumb-item\"><a href=\"/admin\" class=\"text-decoration-none\">Dashboard</a></li><li class=\"breadcrumb-item\"><a href=\"/cluster/ec-shards\" class=\"text-decoration-none\">EC Volumes</a></li><li class=\"breadcrumb-item active\" aria-current=\"page\">Volume ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><div><h1 class=\"h2\"><i class=\"fas fa-th-large me-2\"></i>EC Volume Details</h1><nav aria-label=\"breadcrumb\"><ol class=\"breadcrumb\"><li class=\"breadcrumb-item\"><a href=\"/admin\" class=\"text-decoration-none\">Dashboard</a></li><li class=\"breadcrumb-item\"><a href=\"/storage/ec-shards\" class=\"text-decoration-none\">EC Volumes</a></li><li class=\"breadcrumb-item active\" aria-current=\"page\">Volume ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
weed/admin/view/app/file_browser.templ
+70 -113
View File
@@ -7,6 +7,10 @@ import (
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
)
script changePageSize(path string, lastFileName string) {
window.location.href = '/files?path=' + encodeURIComponent(path) + '&lastFileName=' + encodeURIComponent(lastFileName) + '&limit=' + this.value
}
templ FileBrowser(data dash.FileBrowserData) {
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
<h1 class="h2">
@@ -45,15 +49,15 @@ templ FileBrowser(data dash.FileBrowserData) {
for i, crumb := range data.Breadcrumbs {
if i == len(data.Breadcrumbs)-1 {
<li class="breadcrumb-item active" aria-current="page">
<i class="fas fa-folder me-1"></i>{ crumb.Name }
<a href={ templ.SafeURL(fmt.Sprintf("/files?path=%s", crumb.Path)) } class="text-decoration-none">
{ crumb.Name }
</a>
</li>
} else {
<li class="breadcrumb-item">
<a href={ templ.SafeURL(fmt.Sprintf("/files?path=%s", crumb.Path)) } class="text-decoration-none">
if crumb.Name == "Root" {
<i class="fas fa-home me-1"></i>
} else {
<i class="fas fa-folder me-1"></i>
}
{ crumb.Name }
</a>
@@ -63,110 +67,51 @@ templ FileBrowser(data dash.FileBrowserData) {
</ol>
</nav>
<!-- Summary Cards -->
<div class="row mb-4">
<div class="col-xl-3 col-md-6 mb-4">
<div class="card border-left-primary shadow h-100 py-2">
<div class="card-body">
<div class="row no-gutters align-items-center">
<div class="col mr-2">
<div class="text-xs font-weight-bold text-primary text-uppercase mb-1">
Total Entries
</div>
<div class="h5 mb-0 font-weight-bold text-gray-800">
{ fmt.Sprintf("%d", data.TotalEntries) }
</div>
</div>
<div class="col-auto">
<i class="fas fa-list fa-2x text-gray-300"></i>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3 col-md-6 mb-4">
<div class="card border-left-success shadow h-100 py-2">
<div class="card-body">
<div class="row no-gutters align-items-center">
<div class="col mr-2">
<div class="text-xs font-weight-bold text-success text-uppercase mb-1">
Directories
</div>
<div class="h5 mb-0 font-weight-bold text-gray-800">
{ fmt.Sprintf("%d", countDirectories(data.Entries)) }
</div>
</div>
<div class="col-auto">
<i class="fas fa-folder fa-2x text-gray-300"></i>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3 col-md-6 mb-4">
<div class="card border-left-info shadow h-100 py-2">
<div class="card-body">
<div class="row no-gutters align-items-center">
<div class="col mr-2">
<div class="text-xs font-weight-bold text-info text-uppercase mb-1">
Files
</div>
<div class="h5 mb-0 font-weight-bold text-gray-800">
{ fmt.Sprintf("%d", countFiles(data.Entries)) }
</div>
</div>
<div class="col-auto">
<i class="fas fa-file fa-2x text-gray-300"></i>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3 col-md-6 mb-4">
<div class="card border-left-warning shadow h-100 py-2">
<div class="card-body">
<div class="row no-gutters align-items-center">
<div class="col mr-2">
<div class="text-xs font-weight-bold text-warning text-uppercase mb-1">
Total Size
</div>
<div class="h5 mb-0 font-weight-bold text-gray-800">
{ formatBytes(data.TotalSize) }
</div>
</div>
<div class="col-auto">
<i class="fas fa-hdd fa-2x text-gray-300"></i>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- File Listing -->
<div class="card shadow mb-4">
<div class="card-header py-3 d-flex justify-content-between align-items-center">
<div class="card-header py-3 d-flex justify-content-between align-items-center flex-wrap">
<h6 class="m-0 font-weight-bold text-primary">
<i class="fas fa-folder-open me-2"></i>
if data.CurrentPath == "/" {
Root Directory
<a href="/files?path=/" class="text-decoration-none text-primary">Root Directory</a>
} else if data.CurrentPath == "/buckets" {
Object Store Buckets Directory
<a href="/files?path=/buckets" class="text-decoration-none text-primary">Object Store Buckets Directory</a>
<a href="/object-store/buckets" class="btn btn-sm btn-outline-primary ms-2">
<i class="fas fa-cube me-1"></i>Manage Buckets
</a>
} else {
{ filepath.Base(data.CurrentPath) }
<a href={ templ.SafeURL(fmt.Sprintf("/files?path=%s", data.CurrentPath)) } class="text-decoration-none text-primary">{ filepath.Base(data.CurrentPath) }</a>
}
</h6>
if data.ParentPath != data.CurrentPath {
<a href={ templ.SafeURL(fmt.Sprintf("/files?path=%s", data.ParentPath)) } class="btn btn-sm btn-outline-secondary">
<i class="fas fa-arrow-up me-1"></i>Up
</a>
}
<!-- Compact pagination controls -->
<div class="d-flex align-items-center gap-2 ms-auto">
<div class="d-flex align-items-center">
<label class="me-1 mb-0 small text-muted">Show:</label>
<select class="form-select form-select-sm" style="width: 70px; font-size: 0.875rem;" onchange={ changePageSize(data.CurrentPath, data.CurrentLastFileName) }>
<option value="20" selected?={ data.PageSize == 20 }>20</option>
<option value="50" selected?={ data.PageSize == 50 }>50</option>
<option value="100" selected?={ data.PageSize == 100 }>100</option>
<option value="200" selected?={ data.PageSize == 200 }>200</option>
</select>
</div>
<div class="btn-group btn-group-sm" role="group">
if data.HasNextPage {
<a href={ templ.SafeURL(fmt.Sprintf("/files?path=%s&lastFileName=%s&limit=%d", data.CurrentPath, data.LastFileName, data.PageSize)) } class="btn btn-outline-primary" title="Next page">
Next <i class="fas fa-angle-right"></i>
</a>
} else {
<button class="btn btn-outline-secondary" disabled title="Next page">
Next <i class="fas fa-angle-right"></i>
</button>
}
if data.ParentPath != data.CurrentPath {
<a href={ templ.SafeURL(fmt.Sprintf("/files?path=%s", data.ParentPath)) } class="btn btn-outline-secondary" title="Go up one directory">
<i class="fas fa-arrow-up"></i> Up
</a>
}
</div>
</div>
</div>
<div class="card-body">
if len(data.Entries) > 0 {
@@ -264,6 +209,36 @@ templ FileBrowser(data dash.FileBrowserData) {
</div>
<!-- Last Updated -->
<!-- Pagination Controls (Bottom) -->
<div class="row mb-3">
<div class="col-md-6">
<div class="d-flex align-items-center">
<label class="me-2 mb-0">Show:</label>
<select class="form-select form-select-sm" style="width: auto;" onchange={ changePageSize(data.CurrentPath, data.CurrentLastFileName) }>
<option value="20" selected?={ data.PageSize == 20 }>20</option>
<option value="50" selected?={ data.PageSize == 50 }>50</option>
<option value="100" selected?={ data.PageSize == 100 }>100</option>
<option value="200" selected?={ data.PageSize == 200 }>200</option>
</select>
<span class="ms-2 text-muted">entries per page</span>
</div>
</div>
<div class="col-md-6 text-end">
<div class="btn-group btn-group-sm" role="group">
if data.HasNextPage {
<a href={ templ.SafeURL(fmt.Sprintf("/files?path=%s&lastFileName=%s&limit=%d", data.CurrentPath, data.LastFileName, data.PageSize)) } class="btn btn-outline-primary">
Next <i class="fas fa-angle-right ms-1"></i>
</a>
} else {
<button class="btn btn-outline-secondary" disabled>
Next <i class="fas fa-angle-right ms-1"></i>
</button>
}
</div>
</div>
</div>
<div class="row">
<div class="col-12">
<small class="text-muted">
@@ -732,25 +707,7 @@ templ FileBrowser(data dash.FileBrowserData) {
</script>
}
func countDirectories(entries []dash.FileEntry) int {
count := 0
for _, entry := range entries {
if entry.IsDirectory {
count++
}
}
return count
}
func countFiles(entries []dash.FileEntry) int {
count := 0
for _, entry := range entries {
if !entry.IsDirectory {
count++
}
}
return count
}
func getFileIcon(mime string) string {
switch {
weed/admin/view/app/file_browser_templ.go
+303 -170
View File
File diff suppressed because one or more lines are too long
weed/admin/view/app/maintenance_queue.templ
+4 -18
View File
@@ -320,14 +320,14 @@ templ MaintenanceQueue(data *maintenance.MaintenanceQueueData) {
.then(response => response.json())
.then(data => {
if (data.success) {
alert('Maintenance scan triggered successfully');
showToast('Success', 'Maintenance scan triggered successfully', 'success');
setTimeout(() => window.location.reload(), 2000);
} else {
alert('Failed to trigger scan: ' + (data.error || 'Unknown error'));
showToast('Error', 'Failed to trigger scan: ' + (data.error || 'Unknown error'), 'danger');
}
})
.catch(error => {
alert('Error: ' + error.message);
showToast('Error', 'Error: ' + error.message, 'danger');
});
};
@@ -412,18 +412,4 @@ func formatDuration(d time.Duration) string {
}
}
func formatTimeAgo(t time.Time) string {
duration := time.Since(t)
if duration < time.Minute {
return "just now"
} else if duration < time.Hour {
minutes := int(duration.Minutes())
return fmt.Sprintf("%dm ago", minutes)
} else if duration < 24*time.Hour {
hours := int(duration.Hours())
return fmt.Sprintf("%dh ago", hours)
} else {
days := int(duration.Hours() / 24)
return fmt.Sprintf("%dd ago", days)
}
}
weed/admin/view/app/maintenance_queue_templ.go
+1 -17
View File
@@ -610,7 +610,7 @@ func MaintenanceQueue(data *maintenance.MaintenanceQueueData) templ.Component {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "</div></div></div></div></div><script>\n // Debug output to browser console\n console.log(\"DEBUG: Maintenance Queue Template loaded\");\n \n // Auto-refresh every 10 seconds\n setInterval(function() {\n if (!document.hidden) {\n window.location.reload();\n }\n }, 10000);\n\n window.triggerScan = function() {\n console.log(\"triggerScan called\");\n fetch('/api/maintenance/scan', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n }\n })\n .then(response => response.json())\n .then(data => {\n if (data.success) {\n alert('Maintenance scan triggered successfully');\n setTimeout(() => window.location.reload(), 2000);\n } else {\n alert('Failed to trigger scan: ' + (data.error || 'Unknown error'));\n }\n })\n .catch(error => {\n alert('Error: ' + error.message);\n });\n };\n\n window.refreshPage = function() {\n console.log(\"refreshPage called\");\n window.location.reload();\n };\n\n window.navigateToTask = function(element) {\n const taskId = element.getAttribute('data-task-id');\n if (taskId) {\n window.location.href = '/maintenance/tasks/' + taskId;\n }\n };\n </script>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "</div></div></div></div></div><script>\n // Debug output to browser console\n console.log(\"DEBUG: Maintenance Queue Template loaded\");\n \n // Auto-refresh every 10 seconds\n setInterval(function() {\n if (!document.hidden) {\n window.location.reload();\n }\n }, 10000);\n\n window.triggerScan = function() {\n console.log(\"triggerScan called\");\n fetch('/api/maintenance/scan', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n }\n })\n .then(response => response.json())\n .then(data => {\n if (data.success) {\n showToast('Success', 'Maintenance scan triggered successfully', 'success');\n setTimeout(() => window.location.reload(), 2000);\n } else {\n showToast('Error', 'Failed to trigger scan: ' + (data.error || 'Unknown error'), 'danger');\n }\n })\n .catch(error => {\n showToast('Error', 'Error: ' + error.message, 'danger');\n });\n };\n\n window.refreshPage = function() {\n console.log(\"refreshPage called\");\n window.location.reload();\n };\n\n window.navigateToTask = function(element) {\n const taskId = element.getAttribute('data-task-id');\n if (taskId) {\n window.location.href = '/maintenance/tasks/' + taskId;\n }\n };\n </script>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -857,20 +857,4 @@ func formatDuration(d time.Duration) string {
}
}
func formatTimeAgo(t time.Time) string {
duration := time.Since(t)
if duration < time.Minute {
return "just now"
} else if duration < time.Hour {
minutes := int(duration.Minutes())
return fmt.Sprintf("%dm ago", minutes)
} else if duration < 24*time.Hour {
hours := int(duration.Hours())
return fmt.Sprintf("%dh ago", hours)
} else {
days := int(duration.Hours() / 24)
return fmt.Sprintf("%dd ago", days)
}
}
var _ = templruntime.GeneratedTemplate
weed/admin/view/app/s3_buckets.templ
+15 -11
View File
@@ -164,14 +164,16 @@ templ S3Buckets(data dash.S3BucketsData) {
}
</td>
<td>
if bucket.VersioningEnabled {
if bucket.VersioningStatus == "Enabled" {
<span class="badge bg-success">
<i class="fas fa-check me-1"></i>Enabled
</span>
} else {
<span class="badge bg-secondary">
<i class="fas fa-times me-1"></i>Disabled
} else if bucket.VersioningStatus == "Suspended" {
<span class="badge bg-warning">
<i class="fas fa-pause me-1"></i>Suspended
</span>
} else {
<span class="text-muted">Not configured</span>
}
</td>
<td>
@@ -185,9 +187,7 @@ templ S3Buckets(data dash.S3BucketsData) {
</div>
</div>
} else {
<span class="badge bg-secondary">
<i class="fas fa-unlock me-1"></i>Disabled
</span>
<span class="text-muted">Not configured</span>
}
</td>
<td>
@@ -1044,9 +1044,11 @@ templ S3Buckets(data dash.S3BucketsData) {
'<tr>' +
'<td><strong>Versioning:</strong></td>' +
'<td>' +
(bucket.versioning_enabled ?
(bucket.versioning_status === 'Enabled' ?
'<span class="badge bg-success"><i class="fas fa-check me-1"></i>Enabled</span>' :
'<span class="badge bg-secondary"><i class="fas fa-times me-1"></i>Disabled</span>'
bucket.versioning_status === 'Suspended' ?
'<span class="badge bg-warning"><i class="fas fa-pause me-1"></i>Suspended</span>' :
'<span class="text-muted">Not configured</span>'
) +
'</td>' +
'</tr>' +
@@ -1055,8 +1057,10 @@ templ S3Buckets(data dash.S3BucketsData) {
'<td>' +
(bucket.object_lock_enabled ?
'<span class="badge bg-warning"><i class="fas fa-lock me-1"></i>Enabled</span>' +
'<br><small class="text-muted">' + escapeHtml(bucket.object_lock_mode) + ' • ' + bucket.object_lock_duration + ' days</small>' :
'<span class="badge bg-secondary"><i class="fas fa-unlock me-1"></i>Disabled</span>'
(bucket.object_lock_mode && bucket.object_lock_duration > 0 ?
'<br><small class="text-muted">' + escapeHtml(bucket.object_lock_mode) + ' • ' + bucket.object_lock_duration + ' days</small>' :
'') :
'<span class="text-muted">Not configured</span>'
) +
'</td>' +
'</tr>' +
weed/admin/view/app/s3_buckets_templ.go
+26 -21
View File
File diff suppressed because one or more lines are too long
weed/admin/view/app/service_accounts.templ
+653
View File
@@ -0,0 +1,653 @@
package app
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
)
templ ServiceAccounts(data dash.ServiceAccountsData) {
<div class="container-fluid">
<!-- Page Header -->
<div class="d-sm-flex align-items-center justify-content-between mb-4">
<div>
<h1 class="h3 mb-0 text-gray-800">
<i class="fas fa-robot me-2"></i>Service Accounts
</h1>
<p class="mb-0 text-muted">Manage application credentials for automated processes</p>
</div>
<div class="d-flex gap-2">
<button type="button" class="btn btn-primary"
data-bs-toggle="modal"
data-bs-target="#createServiceAccountModal">
<i class="fas fa-plus me-1"></i>Create Service Account
</button>
</div>
</div>
<!-- Summary Cards -->
<div class="row mb-4">
<div class="col-xl-3 col-md-6 mb-4">
<div class="card border-left-primary shadow h-100 py-2">
<div class="card-body">
<div class="row no-gutters align-items-center">
<div class="col mr-2">
<div class="text-xs font-weight-bold text-primary text-uppercase mb-1">
Total Service Accounts
</div>
<div class="h5 mb-0 font-weight-bold text-gray-800">
{fmt.Sprintf("%d", data.TotalAccounts)}
</div>
</div>
<div class="col-auto">
<i class="fas fa-id-card fa-2x text-gray-300"></i>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3 col-md-6 mb-4">
<div class="card border-left-success shadow h-100 py-2">
<div class="card-body">
<div class="row no-gutters align-items-center">
<div class="col mr-2">
<div class="text-xs font-weight-bold text-success text-uppercase mb-1">
Active Accounts
</div>
<div class="h5 mb-0 font-weight-bold text-gray-800">
{fmt.Sprintf("%d", data.ActiveAccounts)}
</div>
</div>
<div class="col-auto">
<i class="fas fa-check-circle fa-2x text-gray-300"></i>
</div>
</div>
</div>
</div>
</div>
<div class="col-xl-3 col-md-6 mb-4">
<div class="card border-left-info shadow h-100 py-2">
<div class="card-body">
<div class="row no-gutters align-items-center">
<div class="col mr-2">
<div class="text-xs font-weight-bold text-info text-uppercase mb-1">
Last Updated
</div>
<div class="h6 mb-0 font-weight-bold text-gray-800">
{data.LastUpdated.Format("15:04")}
</div>
</div>
<div class="col-auto">
<i class="fas fa-clock fa-2x text-gray-300"></i>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- Service Accounts Table -->
<div class="row">
<div class="col-12">
<div class="card shadow mb-4">
<div class="card-header py-3 d-flex flex-row align-items-center justify-content-between">
<h6 class="m-0 font-weight-bold text-primary">
<i class="fas fa-robot me-2"></i>Service Accounts
</h6>
</div>
<div class="card-body">
<div class="table-responsive">
<table class="table table-hover" width="100%" cellspacing="0" id="serviceAccountsTable">
<thead>
<tr>
<th>ID</th>
<th>Parent User</th>
<th>Access Key</th>
<th>Status</th>
<th>Created</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
for _, sa := range data.ServiceAccounts {
<tr>
<td>
<div class="d-flex align-items-center">
<i class="fas fa-robot me-2 text-muted"></i>
<code>{sa.ID}</code>
</div>
</td>
<td>
<i class="fas fa-user me-1 text-muted"></i>
{sa.ParentUser}
</td>
<td>
<code class="text-muted">{sa.AccessKeyId}</code>
</td>
<td>
if sa.Status == "Active" {
<span class="badge bg-success">Active</span>
} else {
<span class="badge bg-secondary">Inactive</span>
}
</td>
<td>{sa.CreateDate.Format("2006-01-02")}</td>
<td>
<div class="btn-group btn-group-sm" role="group">
<button type="button" class="btn btn-outline-info"
data-action="show-sa-details" data-sa-id={ sa.ID }>
<i class="fas fa-info-circle"></i>
</button>
<button type="button" class="btn btn-outline-primary"
data-action="toggle-sa-status" data-sa-id={ sa.ID } data-current-status={ sa.Status }>
if sa.Status == "Active" {
<i class="fas fa-pause"></i>
} else {
<i class="fas fa-play"></i>
}
</button>
<button type="button" class="btn btn-outline-danger"
data-action="delete-sa" data-sa-id={ sa.ID }>
<i class="fas fa-trash"></i>
</button>
</div>
</td>
</tr>
}
if len(data.ServiceAccounts) == 0 {
<tr>
<td colspan="6" class="text-center text-muted py-4">
<i class="fas fa-robot fa-3x mb-3 text-muted"></i>
<div>
<h5>No service accounts found</h5>
<p>Create your first service account for automated processes.</p>
</div>
</td>
</tr>
}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
<!-- Last Updated -->
<div class="row">
<div class="col-12">
<small class="text-muted">
<i class="fas fa-clock me-1"></i>
Last updated: {data.LastUpdated.Format("2006-01-02 15:04:05")}
</small>
</div>
</div>
</div>
<!-- Create Service Account Modal -->
<div class="modal fade" id="createServiceAccountModal" tabindex="-1" role="dialog">
<div class="modal-dialog" role="document">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">
<i class="fas fa-plus me-2"></i>Create Service Account
</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
</div>
<div class="modal-body">
<form id="createSAForm">
<div class="mb-3">
<label for="parentUser" class="form-label">Parent User *</label>
<select class="form-select" id="parentUser" name="parent_user" required>
<option value="">-- Select a user --</option>
for _, user := range data.AvailableUsers {
<option value={ user }>{ user }</option>
}
</select>
<small class="form-text text-muted">The service account will inherit permissions from this user</small>
</div>
<div class="mb-3">
<label for="description" class="form-label">Description</label>
<textarea class="form-control" id="description" name="description" rows="2"
placeholder="What is this service account used for?"></textarea>
</div>
<div class="mb-3">
<label for="expiration" class="form-label">Expiration (optional)</label>
<input type="datetime-local" class="form-control" id="expiration" name="expiration">
<small class="form-text text-muted">Leave empty for no expiration</small>
</div>
</form>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
<button type="button" class="btn btn-primary" onclick="handleCreateServiceAccount()">Create</button>
</div>
</div>
</div>
</div>
<!-- Service Account Details Modal -->
<div class="modal fade" id="saDetailsModal" tabindex="-1" role="dialog">
<div class="modal-dialog modal-lg" role="document">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title">
<i class="fas fa-robot me-2"></i>Service Account Details
</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal"></button>
</div>
<div class="modal-body" id="saDetailsContent">
<!-- Content will be loaded dynamically -->
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<!-- Credentials Display Modal -->
<div class="modal fade" id="credentialsModal" tabindex="-1" role="dialog" data-bs-backdrop="static" data-bs-keyboard="false">
<div class="modal-dialog modal-lg" role="document">
<div class="modal-content">
<div class="modal-header bg-success text-white">
<h5 class="modal-title">
<i class="fas fa-check-circle me-2"></i>Service Account Created Successfully
</h5>
</div>
<div class="modal-body">
<div class="alert alert-warning">
<i class="fas fa-exclamation-triangle me-2"></i>
<strong>Important:</strong> This is the only time you will see the secret access key. Please save it securely.
</div>
<div class="mb-4">
<h6 class="text-muted mb-3">AWS CLI Configuration</h6>
<p class="text-muted small">Use these credentials to configure AWS CLI or SDKs:</p>
<div class="mb-3">
<label class="form-label fw-bold">AWS_ACCESS_KEY_ID</label>
<div class="input-group">
<input type="text" class="form-control font-monospace" id="displayAccessKey" readonly>
<button class="btn btn-outline-secondary" type="button" onclick="copyToClipboard(event, 'displayAccessKey')">
<i class="fas fa-copy"></i> Copy
</button>
</div>
</div>
<div class="mb-3">
<label class="form-label fw-bold">AWS_SECRET_ACCESS_KEY</label>
<div class="input-group">
<input type="text" class="form-control font-monospace" id="displaySecretKey" readonly>
<button class="btn btn-outline-secondary" type="button" onclick="copyToClipboard(event, 'displaySecretKey')">
<i class="fas fa-copy"></i> Copy
</button>
</div>
</div>
</div>
<div class="bg-light p-3 rounded">
<h6 class="text-muted mb-2">Example AWS CLI Usage:</h6>
<div class="font-monospace small">
<div>export AWS_ACCESS_KEY_ID=<span id="exampleAccessKey"></span></div>
<div>export AWS_SECRET_ACCESS_KEY=<span id="exampleSecretKey"></span></div>
<div>export AWS_ENDPOINT_URL=http://localhost:8333</div>
<div class="mt-2"># List buckets</div>
<div>aws s3 ls</div>
<div class="mt-2"># Upload a file</div>
<div>aws s3 cp myfile.txt s3://mybucket/</div>
</div>
</div>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-primary" onclick="closeCredentialsModal()">
<i class="fas fa-check me-1"></i>I have saved the credentials
</button>
</div>
</div>
</div>
</div>
<!-- JavaScript for service account management -->
<script>
document.addEventListener('DOMContentLoaded', function() {
document.addEventListener('click', function(e) {
const button = e.target.closest('[data-action]');
if (!button) return;
const action = button.getAttribute('data-action');
const saId = button.getAttribute('data-sa-id');
switch (action) {
case 'show-sa-details':
showSADetails(saId);
break;
case 'toggle-sa-status':
toggleSAStatus(saId, button.getAttribute('data-current-status'));
break;
case 'delete-sa':
deleteSA(saId);
break;
}
});
});
async function showSADetails(id) {
try {
const response = await fetch(`/api/service-accounts/${id}`);
if (response.ok) {
const sa = await response.json();
document.getElementById('saDetailsContent').innerHTML = createSADetailsContent(sa);
const modal = new bootstrap.Modal(document.getElementById('saDetailsModal'));
modal.show();
} else {
showErrorMessage('Failed to load service account details');
}
} catch (error) {
console.error('Error loading service account details:', error);
showErrorMessage('Failed to load service account details');
}
}
async function toggleSAStatus(id, currentStatus) {
const newStatus = currentStatus === 'Active' ? 'Inactive' : 'Active';
try {
const response = await fetch(`/api/service-accounts/${id}`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ status: newStatus })
});
if (response.ok) {
showSuccessMessage(`Service account ${newStatus === 'Active' ? 'activated' : 'deactivated'}`);
setTimeout(() => window.location.reload(), 1000);
} else {
const error = await response.json();
showErrorMessage('Failed to update status: ' + (error.error || 'Unknown error'));
}
} catch (error) {
console.error('Error updating status:', error);
showErrorMessage('Failed to update status: ' + error.message);
}
}
async function deleteSA(id) {
if (confirm('Are you sure you want to delete this service account? This action cannot be undone.')) {
try {
const response = await fetch(`/api/service-accounts/${id}`, {
method: 'DELETE'
});
if (response.ok) {
showSuccessMessage('Service account deleted successfully');
setTimeout(() => window.location.reload(), 1000);
} else {
const error = await response.json();
showErrorMessage('Failed to delete: ' + (error.error || 'Unknown error'));
}
} catch (error) {
console.error('Error deleting service account:', error);
showErrorMessage('Failed to delete: ' + error.message);
}
}
}
async function handleCreateServiceAccount() {
const form = document.getElementById('createSAForm');
const formData = new FormData(form);
const saData = {
parent_user: formData.get('parent_user'),
description: formData.get('description')
};
// Handle expiration if set
const expiration = formData.get('expiration');
if (expiration) {
// Validate the date before using it
const date = new Date(expiration);
const now = new Date();
if (isNaN(date.getTime())) {
showErrorMessage('Invalid expiration date format');
return;
}
// Ensure expiration is in the future
if (date <= now) {
showErrorMessage('Expiration date must be in the future');
return;
}
saData.expiration = date.toISOString();
}
try {
const response = await fetch('/api/service-accounts', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(saData)
});
if (response.ok) {
const result = await response.json();
// Hide create modal
const createModal = bootstrap.Modal.getInstance(document.getElementById('createServiceAccountModal'));
createModal.hide();
form.reset();
// Show credentials if returned
if (result.service_account && result.service_account.secret_access_key) {
showCredentials(result.service_account);
} else {
showSuccessMessage('Service account created successfully');
setTimeout(() => window.location.reload(), 1000);
}
} else {
const error = await response.json();
showErrorMessage('Failed to create service account: ' + (error.error || 'Unknown error'));
}
} catch (error) {
console.error('Error creating service account:', error);
showErrorMessage('Failed to create service account: ' + error.message);
}
}
function showCredentials(serviceAccount) {
// Populate the credentials modal
document.getElementById('displayAccessKey').value = serviceAccount.access_key_id;
document.getElementById('displaySecretKey').value = serviceAccount.secret_access_key;
document.getElementById('exampleAccessKey').textContent = serviceAccount.access_key_id;
document.getElementById('exampleSecretKey').textContent = serviceAccount.secret_access_key;
// Show the modal
const credentialsModal = new bootstrap.Modal(document.getElementById('credentialsModal'));
credentialsModal.show();
}
function closeCredentialsModal() {
const modal = bootstrap.Modal.getInstance(document.getElementById('credentialsModal'));
modal.hide();
// Reload to show the new service account in the list
setTimeout(() => window.location.reload(), 500);
}
function copyToClipboard(event, elementId) {
const element = document.getElementById(elementId);
// Use modern Clipboard API if available
if (navigator.clipboard) {
navigator.clipboard.writeText(element.value).then(() => {
// Success feedback could be added here
}).catch(err => {
console.warn('Clipboard API failed:', err);
// Fallback
element.select();
document.execCommand('copy');
});
} else {
// Fallback for older browsers
element.select();
document.execCommand('copy');
}
// Visual feedback
const button = event.target.closest('button');
const originalHTML = button.innerHTML;
button.innerHTML = '<i class="fas fa-check"></i>';
button.classList.remove('btn-outline-secondary');
button.classList.add('btn-success');
setTimeout(() => {
button.innerHTML = originalHTML;
button.classList.remove('btn-success');
button.classList.add('btn-outline-secondary');
}, 1000);
}
function createSADetailsContent(sa) {
// Create DOM elements safely to prevent XSS
const container = document.createElement('div');
container.className = 'row';
// Basic Information column
const col1 = document.createElement('div');
col1.className = 'col-md-6';
const h6_1 = document.createElement('h6');
h6_1.className = 'text-muted';
h6_1.textContent = 'Basic Information';
col1.appendChild(h6_1);
const table1 = document.createElement('table');
table1.className = 'table table-sm';
// ID row
const idRow = document.createElement('tr');
idRow.innerHTML = '<td><strong>ID:</strong></td><td><code></code></td>';
idRow.querySelector('code').textContent = sa.id || '';
table1.appendChild(idRow);
// Parent User row
const parentRow = document.createElement('tr');
parentRow.innerHTML = '<td><strong>Parent User:</strong></td><td></td>';
parentRow.querySelectorAll('td')[1].textContent = sa.parent_user || '';
table1.appendChild(parentRow);
// Access Key row
const keyRow = document.createElement('tr');
keyRow.innerHTML = '<td><strong>Access Key:</strong></td><td><code></code></td>';
keyRow.querySelector('code').textContent = sa.access_key_id || '';
table1.appendChild(keyRow);
// Status row
const statusRow = document.createElement('tr');
const statusTd1 = document.createElement('td');
statusTd1.innerHTML = '<strong>Status:</strong>';
const statusTd2 = document.createElement('td');
const statusBadge = document.createElement('span');
statusBadge.className = sa.status === 'Active' ? 'badge bg-success' : 'badge bg-secondary';
statusBadge.textContent = sa.status || 'Unknown';
statusTd2.appendChild(statusBadge);
statusRow.appendChild(statusTd1);
statusRow.appendChild(statusTd2);
table1.appendChild(statusRow);
col1.appendChild(table1);
container.appendChild(col1);
// Details column
const col2 = document.createElement('div');
col2.className = 'col-md-6';
const h6_2 = document.createElement('h6');
h6_2.className = 'text-muted';
h6_2.textContent = 'Details';
col2.appendChild(h6_2);
const table2 = document.createElement('table');
table2.className = 'table table-sm';
// Description row
const descRow = document.createElement('tr');
descRow.innerHTML = '<td><strong>Description:</strong></td><td></td>';
descRow.querySelectorAll('td')[1].textContent = sa.description || 'Not set';
table2.appendChild(descRow);
// Created row
const createdRow = document.createElement('tr');
createdRow.innerHTML = '<td><strong>Created:</strong></td><td></td>';
try {
createdRow.querySelectorAll('td')[1].textContent = new Date(sa.create_date).toLocaleString();
} catch (e) {
createdRow.querySelectorAll('td')[1].textContent = 'Invalid date';
}
table2.appendChild(createdRow);
// Expiration row
const expRow = document.createElement('tr');
expRow.innerHTML = '<td><strong>Expires:</strong></td><td></td>';
expRow.querySelectorAll('td')[1].textContent = sa.expiration || 'Never';
table2.appendChild(expRow);
col2.appendChild(table2);
container.appendChild(col2);
return container.outerHTML;
}
function showSuccessMessage(message) {
showToast(message, 'success');
}
function showErrorMessage(message) {
showToast(message, 'danger');
}
function showToast(message, type) {
// Create toast container if it doesn't exist
let toastContainer = document.getElementById('toastContainer');
if (!toastContainer) {
toastContainer = document.createElement('div');
toastContainer.id = 'toastContainer';
toastContainer.className = 'toast-container position-fixed top-0 end-0 p-3';
toastContainer.style.zIndex = '9999';
document.body.appendChild(toastContainer);
}
// Create toast element
const toastId = 'toast-' + Date.now();
const toastHTML = `
<div id="${toastId}" class="toast align-items-center text-white bg-${type} border-0" role="alert" aria-live="assertive" aria-atomic="true">
<div class="d-flex">
<div class="toast-body">
${escapeHtml(message)}
</div>
<button type="button" class="btn-close btn-close-white me-2 m-auto" data-bs-dismiss="toast" aria-label="Close"></button>
</div>
</div>
`;
toastContainer.insertAdjacentHTML('beforeend', toastHTML);
const toastElement = document.getElementById(toastId);
const toast = new bootstrap.Toast(toastElement, { autohide: true, delay: 5000 });
toast.show();
// Remove toast element after it's hidden
toastElement.addEventListener('hidden.bs.toast', () => {
toastElement.remove();
});
}
function escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
</script>
}
weed/admin/view/app/service_accounts_templ.go
+283
View File
File diff suppressed because one or more lines are too long
weed/admin/view/app/volume_details.templ
+4 -4
View File
@@ -15,7 +15,7 @@ templ VolumeDetails(data dash.VolumeDetailsData) {
<nav aria-label="breadcrumb">
<ol class="breadcrumb">
<li class="breadcrumb-item"><a href="/admin" class="text-decoration-none">Dashboard</a></li>
<li class="breadcrumb-item"><a href="/cluster/volumes" class="text-decoration-none">Volumes</a></li>
<li class="breadcrumb-item"><a href="/storage/volumes" class="text-decoration-none">Volumes</a></li>
<li class="breadcrumb-item active" aria-current="page">Volume {fmt.Sprintf("%d", data.Volume.Id)}</li>
</ol>
</nav>
@@ -71,11 +71,11 @@ templ VolumeDetails(data dash.VolumeDetailsData) {
<label class="form-label"><strong>Collection:</strong></label>
<div>
if data.Volume.Collection == "" {
<a href={templ.SafeURL("/cluster/volumes?collection=default")} class="text-decoration-none">
<a href={templ.SafeURL("/storage/volumes?collection=default")} class="text-decoration-none">
<span class="badge bg-secondary">default</span>
</a>
} else {
<a href={templ.SafeURL(fmt.Sprintf("/cluster/volumes?collection=%s", data.Volume.Collection))} class="text-decoration-none">
<a href={templ.SafeURL(fmt.Sprintf("/storage/volumes?collection=%s", data.Volume.Collection))} class="text-decoration-none">
<span class="badge bg-secondary">{data.Volume.Collection}</span>
</a>
}
@@ -329,7 +329,7 @@ templ VolumeDetails(data dash.VolumeDetailsData) {
<td>{fmt.Sprintf("%d", replica.FileCount)}</td>
<td><span class="badge bg-info">Replica</span></td>
<td>
<a href={templ.SafeURL(fmt.Sprintf("/cluster/volumes/%d/%s", replica.Id, replica.Server))} class="btn btn-sm btn-outline-primary">
<a href={templ.SafeURL(fmt.Sprintf("/storage/volumes/%d/%s", replica.Id, replica.Server))} class="btn btn-sm btn-outline-primary">
<i class="fas fa-eye me-1"></i>View
</a>
</td>
weed/admin/view/app/volume_details_templ.go
+4 -4
View File
@@ -35,7 +35,7 @@ func VolumeDetails(data dash.VolumeDetailsData) templ.Component {
templ_7745c5c3_Var1 = templ.NopComponent
}
ctx = templ.ClearChildren(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><div><h1 class=\"h2\"><i class=\"fas fa-database me-2\"></i>Volume Details</h1><nav aria-label=\"breadcrumb\"><ol class=\"breadcrumb\"><li class=\"breadcrumb-item\"><a href=\"/admin\" class=\"text-decoration-none\">Dashboard</a></li><li class=\"breadcrumb-item\"><a href=\"/cluster/volumes\" class=\"text-decoration-none\">Volumes</a></li><li class=\"breadcrumb-item active\" aria-current=\"page\">Volume ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><div><h1 class=\"h2\"><i class=\"fas fa-database me-2\"></i>Volume Details</h1><nav aria-label=\"breadcrumb\"><ol class=\"breadcrumb\"><li class=\"breadcrumb-item\"><a href=\"/admin\" class=\"text-decoration-none\">Dashboard</a></li><li class=\"breadcrumb-item\"><a href=\"/storage/volumes\" class=\"text-decoration-none\">Volumes</a></li><li class=\"breadcrumb-item active\" aria-current=\"page\">Volume ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -123,7 +123,7 @@ func VolumeDetails(data dash.VolumeDetailsData) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var8 templ.SafeURL
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL("/cluster/volumes?collection=default"))
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL("/storage/volumes?collection=default"))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/volume_details.templ`, Line: 74, Col: 101}
}
@@ -141,7 +141,7 @@ func VolumeDetails(data dash.VolumeDetailsData) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var9 templ.SafeURL
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/cluster/volumes?collection=%s", data.Volume.Collection)))
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/storage/volumes?collection=%s", data.Volume.Collection)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/volume_details.templ`, Line: 78, Col: 133}
}
@@ -635,7 +635,7 @@ func VolumeDetails(data dash.VolumeDetailsData) templ.Component {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var40 templ.SafeURL
templ_7745c5c3_Var40, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/cluster/volumes/%d/%s", replica.Id, replica.Server)))
templ_7745c5c3_Var40, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/storage/volumes/%d/%s", replica.Id, replica.Server)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/volume_details.templ`, Line: 332, Col: 137}
}
weed/admin/view/layout/layout.templ
+36 -12
View File
@@ -21,6 +21,12 @@ templ Layout(c *gin.Context, content templ.Component) {
// Detect if we're on a message queue page to keep submenu expanded
isMQPage := strings.HasPrefix(currentPath, "/mq/")
// Detect if we're on a storage page to keep submenu expanded
isStoragePage := strings.HasPrefix(currentPath, "/storage/volumes") || strings.HasPrefix(currentPath, "/storage/ec-shards") || strings.HasPrefix(currentPath, "/storage/collections")
// Detect if we're on a cluster page (but not storage page) to keep submenu expanded
isClusterPage := (strings.HasPrefix(currentPath, "/cluster/masters") || strings.HasPrefix(currentPath, "/cluster/volume-servers") || strings.HasPrefix(currentPath, "/cluster/filers"))
}}
<!DOCTYPE html>
<html lang="en">
@@ -85,11 +91,11 @@ templ Layout(c *gin.Context, content templ.Component) {
</a>
</li>
<li class="nav-item">
<a class="nav-link collapsed" href="#" data-bs-toggle="collapse" data-bs-target="#clusterSubmenu" aria-expanded="false" aria-controls="clusterSubmenu">
<a class={ "nav-link", templ.KV("collapsed", !isClusterPage) } href="#" data-bs-toggle="collapse" data-bs-target="#clusterSubmenu" aria-expanded={ fmt.Sprintf("%t", isClusterPage) } aria-controls="clusterSubmenu">
<i class="fas fa-sitemap me-2"></i>Cluster
<i class="fas fa-chevron-down ms-auto"></i>
</a>
<div class="collapse" id="clusterSubmenu">
<div class={ "collapse", templ.KV("show", isClusterPage) } id="clusterSubmenu">
<ul class="nav flex-column ms-3">
<li class="nav-item">
<a class="nav-link py-2" href="/cluster/masters">
@@ -104,20 +110,30 @@ templ Layout(c *gin.Context, content templ.Component) {
<li class="nav-item">
<a class="nav-link py-2" href="/cluster/filers">
<i class="fas fa-folder-open me-2"></i>Filers
</a>
</a>
</li>
</ul>
</div>
</li>
<li class="nav-item">
<a class="nav-link py-2" href="/cluster/volumes">
<i class="fas fa-database me-2"></i>Volumes
<a class={ "nav-link", templ.KV("collapsed", !isStoragePage) } href="#" data-bs-toggle="collapse" data-bs-target="#storageSubmenu" aria-expanded={ fmt.Sprintf("%t", isStoragePage) } aria-controls="storageSubmenu">
<i class="fas fa-hdd me-2"></i>Storage
<i class="fas fa-chevron-down ms-auto"></i>
</a>
<div class={ "collapse", templ.KV("show", isStoragePage) } id="storageSubmenu">
<ul class="nav flex-column ms-3">
<li class="nav-item">
<a class="nav-link py-2" href="/storage/volumes">
<i class="fas fa-database me-2"></i>Volumes
</a>
</li>
<li class="nav-item">
<a class="nav-link py-2" href="/cluster/ec-shards">
<a class="nav-link py-2" href="/storage/ec-shards">
<i class="fas fa-th-large me-2"></i>EC Volumes
</a>
</li>
<li class="nav-item">
<a class="nav-link py-2" href="/cluster/collections">
<a class="nav-link py-2" href="/storage/collections">
<i class="fas fa-layer-group me-2"></i>Collections
</a>
</li>
@@ -152,6 +168,11 @@ templ Layout(c *gin.Context, content templ.Component) {
<i class="fas fa-users me-2"></i>Users
</a>
</li>
<li class="nav-item">
<a class="nav-link py-2" href="/object-store/service-accounts">
<i class="fas fa-robot me-2"></i>Service Accounts
</a>
</li>
<li class="nav-item">
<a class="nav-link py-2" href="/object-store/policies">
<i class="fas fa-shield-alt me-2"></i>Policies
@@ -218,6 +239,8 @@ templ Layout(c *gin.Context, content templ.Component) {
</div>
}
</li>
<!-- Commented out for later -->
<!--
<li class="nav-item">
<a class="nav-link" href="/metrics">
<i class="fas fa-chart-line me-2"></i>Metrics
@@ -228,10 +251,11 @@ templ Layout(c *gin.Context, content templ.Component) {
<i class="fas fa-file-alt me-2"></i>Logs
</a>
</li>
-->
</ul>
<h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted">
<span>SYSTEM</span>
<span>MAINTENANCE</span>
</h6>
<ul class="nav flex-column">
<li class="nav-item">
@@ -284,22 +308,22 @@ templ Layout(c *gin.Context, content templ.Component) {
<li class="nav-item">
if currentPath == "/maintenance" {
<a class="nav-link active" href="/maintenance">
<i class="fas fa-list me-2"></i>Maintenance Queue
<i class="fas fa-list me-2"></i>Job Queue
</a>
} else {
<a class="nav-link" href="/maintenance">
<i class="fas fa-list me-2"></i>Maintenance Queue
<i class="fas fa-list me-2"></i>Job Queue
</a>
}
</li>
<li class="nav-item">
if currentPath == "/maintenance/workers" {
<a class="nav-link active" href="/maintenance/workers">
<i class="fas fa-user-cog me-2"></i>Maintenance Workers
<i class="fas fa-user-cog me-2"></i>Workers
</a>
} else {
<a class="nav-link" href="/maintenance/workers">
<i class="fas fa-user-cog me-2"></i>Maintenance Workers
<i class="fas fa-user-cog me-2"></i>Workers
</a>
}
</li>
weed/admin/view/layout/layout_templ.go
+236 -116
View File
@@ -48,6 +48,12 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
// Detect if we're on a message queue page to keep submenu expanded
isMQPage := strings.HasPrefix(currentPath, "/mq/")
// Detect if we're on a storage page to keep submenu expanded
isStoragePage := strings.HasPrefix(currentPath, "/storage/volumes") || strings.HasPrefix(currentPath, "/storage/ec-shards") || strings.HasPrefix(currentPath, "/storage/collections")
// Detect if we're on a cluster page (but not storage page) to keep submenu expanded
isClusterPage := (strings.HasPrefix(currentPath, "/cluster/masters") || strings.HasPrefix(currentPath, "/cluster/volume-servers") || strings.HasPrefix(currentPath, "/cluster/filers"))
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>SeaweedFS Admin</title><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><!-- Bootstrap CSS --><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><!-- Font Awesome CSS --><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"><!-- HTMX --><script src=\"/static/js/htmx.min.js\"></script><!-- Custom CSS --><link rel=\"stylesheet\" href=\"/static/css/admin.css\"></head><body><div class=\"container-fluid\"><!-- Header --><header class=\"navbar navbar-expand-lg navbar-dark bg-primary sticky-top\"><div class=\"container-fluid\"><a class=\"navbar-brand fw-bold\" href=\"/admin\"><i class=\"fas fa-server me-2\"></i> SeaweedFS Admin <span class=\"badge bg-warning text-dark ms-2\">ALPHA</span></a> <button class=\"navbar-toggler\" type=\"button\" data-bs-toggle=\"collapse\" data-bs-target=\"#navbarNav\"><span class=\"navbar-toggler-icon\"></span></button><div class=\"collapse navbar-collapse\" id=\"navbarNav\"><ul class=\"navbar-nav ms-auto\"><li class=\"nav-item dropdown\"><a class=\"nav-link dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-user me-1\"></i>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
@@ -55,305 +61,419 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var2 string
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(username)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 61, Col: 73}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 67, Col: 73}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "</a><ul class=\"dropdown-menu\"><li><a class=\"dropdown-item\" href=\"/logout\"><i class=\"fas fa-sign-out-alt me-2\"></i>Logout</a></li></ul></li></ul></div></div></header><div class=\"row g-0\"><!-- Sidebar --><div class=\"col-md-3 col-lg-2 d-md-block bg-light sidebar collapse\"><div class=\"position-sticky pt-3\"><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MAIN</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"/admin\"><i class=\"fas fa-tachometer-alt me-2\"></i>Dashboard</a></li><li class=\"nav-item\"><a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#clusterSubmenu\" aria-expanded=\"false\" aria-controls=\"clusterSubmenu\"><i class=\"fas fa-sitemap me-2\"></i>Cluster <i class=\"fas fa-chevron-down ms-auto\"></i></a><div class=\"collapse\" id=\"clusterSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/cluster/masters\"><i class=\"fas fa-crown me-2\"></i>Masters</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/cluster/volume-servers\"><i class=\"fas fa-server me-2\"></i>Volume Servers</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/cluster/filers\"><i class=\"fas fa-folder-open me-2\"></i>Filers</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/cluster/volumes\"><i class=\"fas fa-database me-2\"></i>Volumes</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/cluster/ec-shards\"><i class=\"fas fa-th-large me-2\"></i>EC Volumes</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/cluster/collections\"><i class=\"fas fa-layer-group me-2\"></i>Collections</a></li></ul></div></li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MANAGEMENT</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"/files\"><i class=\"fas fa-folder me-2\"></i>File Browser</a></li><li class=\"nav-item\"><a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#objectStoreSubmenu\" aria-expanded=\"false\" aria-controls=\"objectStoreSubmenu\"><i class=\"fas fa-cloud me-2\"></i>Object Store <i class=\"fas fa-chevron-down ms-auto\"></i></a><div class=\"collapse\" id=\"objectStoreSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/buckets\"><i class=\"fas fa-cube me-2\"></i>Buckets</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/users\"><i class=\"fas fa-users me-2\"></i>Users</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/policies\"><i class=\"fas fa-shield-alt me-2\"></i>Policies</a></li></ul></div></li><li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "</a><ul class=\"dropdown-menu\"><li><a class=\"dropdown-item\" href=\"/logout\"><i class=\"fas fa-sign-out-alt me-2\"></i>Logout</a></li></ul></li></ul></div></div></header><div class=\"row g-0\"><!-- Sidebar --><div class=\"col-md-3 col-lg-2 d-md-block bg-light sidebar collapse\"><div class=\"position-sticky pt-3\"><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MAIN</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"/admin\"><i class=\"fas fa-tachometer-alt me-2\"></i>Dashboard</a></li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var3 = []any{"nav-link", templ.KV("collapsed", !isClusterPage)}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var3...)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "<a class=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var4 string
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var3).String())
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 1, Col: 0}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#clusterSubmenu\" aria-expanded=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var5 string
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isClusterPage))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 94, Col: 207}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "\" aria-controls=\"clusterSubmenu\"><i class=\"fas fa-sitemap me-2\"></i>Cluster <i class=\"fas fa-chevron-down ms-auto\"></i></a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var6 = []any{"collapse", templ.KV("show", isClusterPage)}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var6...)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<div class=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var7 string
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var6).String())
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 1, Col: 0}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "\" id=\"clusterSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/cluster/masters\"><i class=\"fas fa-crown me-2\"></i>Masters</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/cluster/volume-servers\"><i class=\"fas fa-server me-2\"></i>Volume Servers</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/cluster/filers\"><i class=\"fas fa-folder-open me-2\"></i>Filers</a></li></ul></div></li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var8 = []any{"nav-link", templ.KV("collapsed", !isStoragePage)}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var8...)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "<a class=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var9 string
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var8).String())
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 1, Col: 0}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#storageSubmenu\" aria-expanded=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var10 string
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isStoragePage))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 119, Col: 207}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "\" aria-controls=\"storageSubmenu\"><i class=\"fas fa-hdd me-2\"></i>Storage <i class=\"fas fa-chevron-down ms-auto\"></i></a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var11 = []any{"collapse", templ.KV("show", isStoragePage)}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var11...)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "<div class=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var12 string
templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var11).String())
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 1, Col: 0}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "\" id=\"storageSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/storage/volumes\"><i class=\"fas fa-database me-2\"></i>Volumes</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/storage/ec-shards\"><i class=\"fas fa-th-large me-2\"></i>EC Volumes</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/storage/collections\"><i class=\"fas fa-layer-group me-2\"></i>Collections</a></li></ul></div></li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MANAGEMENT</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"/files\"><i class=\"fas fa-folder me-2\"></i>File Browser</a></li><li class=\"nav-item\"><a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#objectStoreSubmenu\" aria-expanded=\"false\" aria-controls=\"objectStoreSubmenu\"><i class=\"fas fa-cloud me-2\"></i>Object Store <i class=\"fas fa-chevron-down ms-auto\"></i></a><div class=\"collapse\" id=\"objectStoreSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/buckets\"><i class=\"fas fa-cube me-2\"></i>Buckets</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/users\"><i class=\"fas fa-users me-2\"></i>Users</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/service-accounts\"><i class=\"fas fa-robot me-2\"></i>Service Accounts</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/object-store/policies\"><i class=\"fas fa-shield-alt me-2\"></i>Policies</a></li></ul></div></li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isMQPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "<a class=\"nav-link\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"true\" aria-controls=\"messageQueueSubmenu\"><i class=\"fas fa-comments me-2\"></i>Message Queue <i class=\"fas fa-chevron-down ms-auto\"></i></a> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "<a class=\"nav-link\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"true\" aria-controls=\"messageQueueSubmenu\"><i class=\"fas fa-comments me-2\"></i>Message Queue <i class=\"fas fa-chevron-down ms-auto\"></i></a> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "<a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"false\" aria-controls=\"messageQueueSubmenu\"><i class=\"fas fa-comments me-2\"></i>Message Queue <i class=\"fas fa-chevron-down ms-auto\"></i></a> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "<a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#messageQueueSubmenu\" aria-expanded=\"false\" aria-controls=\"messageQueueSubmenu\"><i class=\"fas fa-comments me-2\"></i>Message Queue <i class=\"fas fa-chevron-down ms-auto\"></i></a> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
if isMQPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "<div class=\"collapse show\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "<div class=\"collapse show\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if currentPath == "/mq/brokers" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "<a class=\"nav-link py-2 active\" href=\"/mq/brokers\"><i class=\"fas fa-server me-2\"></i>Brokers</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "<a class=\"nav-link py-2 active\" href=\"/mq/brokers\"><i class=\"fas fa-server me-2\"></i>Brokers</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<a class=\"nav-link py-2\" href=\"/mq/brokers\"><i class=\"fas fa-server me-2\"></i>Brokers</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "<a class=\"nav-link py-2\" href=\"/mq/brokers\"><i class=\"fas fa-server me-2\"></i>Brokers</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "</li><li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if currentPath == "/mq/topics" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "<a class=\"nav-link py-2 active\" href=\"/mq/topics\"><i class=\"fas fa-list-alt me-2\"></i>Topics</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "<a class=\"nav-link py-2 active\" href=\"/mq/topics\"><i class=\"fas fa-list-alt me-2\"></i>Topics</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "<a class=\"nav-link py-2\" href=\"/mq/topics\"><i class=\"fas fa-list-alt me-2\"></i>Topics</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "<a class=\"nav-link py-2\" href=\"/mq/topics\"><i class=\"fas fa-list-alt me-2\"></i>Topics</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</li></ul></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "</li></ul></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "<div class=\"collapse\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/mq/brokers\"><i class=\"fas fa-server me-2\"></i>Brokers</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/mq/topics\"><i class=\"fas fa-list-alt me-2\"></i>Topics</a></li></ul></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "<div class=\"collapse\" id=\"messageQueueSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/mq/brokers\"><i class=\"fas fa-server me-2\"></i>Brokers</a></li><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/mq/topics\"><i class=\"fas fa-list-alt me-2\"></i>Topics</a></li></ul></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "</li><li class=\"nav-item\"><a class=\"nav-link\" href=\"/metrics\"><i class=\"fas fa-chart-line me-2\"></i>Metrics</a></li><li class=\"nav-item\"><a class=\"nav-link\" href=\"/logs\"><i class=\"fas fa-file-alt me-2\"></i>Logs</a></li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>SYSTEM</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "</li><!-- Commented out for later --><!--\n <li class=\"nav-item\">\n <a class=\"nav-link\" href=\"/metrics\">\n <i class=\"fas fa-chart-line me-2\"></i>Metrics\n </a>\n </li>\n <li class=\"nav-item\">\n <a class=\"nav-link\" href=\"/logs\">\n <i class=\"fas fa-file-alt me-2\"></i>Logs\n </a>\n </li>\n --></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MAINTENANCE</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isConfigPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "<a class=\"nav-link\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#configurationSubmenu\" aria-expanded=\"true\" aria-controls=\"configurationSubmenu\"><i class=\"fas fa-cogs me-2\"></i>Configuration <i class=\"fas fa-chevron-down ms-auto\"></i></a> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "<a class=\"nav-link\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#configurationSubmenu\" aria-expanded=\"true\" aria-controls=\"configurationSubmenu\"><i class=\"fas fa-cogs me-2\"></i>Configuration <i class=\"fas fa-chevron-down ms-auto\"></i></a> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "<a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#configurationSubmenu\" aria-expanded=\"false\" aria-controls=\"configurationSubmenu\"><i class=\"fas fa-cogs me-2\"></i>Configuration <i class=\"fas fa-chevron-right ms-auto\"></i></a> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "<a class=\"nav-link collapsed\" href=\"#\" data-bs-toggle=\"collapse\" data-bs-target=\"#configurationSubmenu\" aria-expanded=\"false\" aria-controls=\"configurationSubmenu\"><i class=\"fas fa-cogs me-2\"></i>Configuration <i class=\"fas fa-chevron-right ms-auto\"></i></a> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
if isConfigPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "<div class=\"collapse show\" id=\"configurationSubmenu\"><ul class=\"nav flex-column ms-3\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "<div class=\"collapse show\" id=\"configurationSubmenu\"><ul class=\"nav flex-column ms-3\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
for _, menuItem := range GetConfigurationMenuItems() {
isActiveItem := currentPath == menuItem.URL
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "<li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "<li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isActiveItem {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "<a class=\"nav-link py-2 active\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "<a class=\"nav-link py-2 active\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var3 templ.SafeURL
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
var templ_7745c5c3_Var13 templ.SafeURL
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 258, Col: 117}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 282, Col: 117}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var4 = []any{menuItem.Icon + " me-2"}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var4...)
var templ_7745c5c3_Var14 = []any{menuItem.Icon + " me-2"}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var14...)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 20, "<i class=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "<i class=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var5 string
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var4).String())
var templ_7745c5c3_Var15 string
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var14).String())
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 1, Col: 0}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 21, "\"></i>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "\"></i>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
var templ_7745c5c3_Var16 string
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 259, Col: 109}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 283, Col: 109}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 22, "</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 23, "<a class=\"nav-link py-2\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "<a class=\"nav-link py-2\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var7 templ.SafeURL
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
var templ_7745c5c3_Var17 templ.SafeURL
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 262, Col: 110}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 286, Col: 110}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var8 = []any{menuItem.Icon + " me-2"}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var8...)
var templ_7745c5c3_Var18 = []any{menuItem.Icon + " me-2"}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var18...)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 25, "<i class=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "<i class=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var9 string
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var8).String())
var templ_7745c5c3_Var19 string
templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var18).String())
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 1, Col: 0}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 26, "\"></i>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "\"></i>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var10 string
templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
var templ_7745c5c3_Var20 string
templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 263, Col: 109}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 287, Col: 109}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 27, "</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 28, "</li>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "</li>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 29, "</ul></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "</ul></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 30, "<div class=\"collapse\" id=\"configurationSubmenu\"><ul class=\"nav flex-column ms-3\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "<div class=\"collapse\" id=\"configurationSubmenu\"><ul class=\"nav flex-column ms-3\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
for _, menuItem := range GetConfigurationMenuItems() {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 31, "<li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "<li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var11 templ.SafeURL
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
var templ_7745c5c3_Var21 templ.SafeURL
templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 275, Col: 106}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 299, Col: 106}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 32, "\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var12 = []any{menuItem.Icon + " me-2"}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var12...)
var templ_7745c5c3_Var22 = []any{menuItem.Icon + " me-2"}
templ_7745c5c3_Err = templ.RenderCSSItems(ctx, templ_7745c5c3_Buffer, templ_7745c5c3_Var22...)
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 33, "<i class=\"")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "<i class=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var13 string
templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var12).String())
var templ_7745c5c3_Var23 string
templ_7745c5c3_Var23, templ_7745c5c3_Err = templ.JoinStringErrs(templ.CSSClasses(templ_7745c5c3_Var22).String())
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 1, Col: 0}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var23))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 34, "\"></i>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "\"></i>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var14 string
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
var templ_7745c5c3_Var24 string
templ_7745c5c3_Var24, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 276, Col: 105}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 300, Col: 105}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var24))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 35, "</a></li>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, "</a></li>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "</ul></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, "</ul></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 37, "</li><li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if currentPath == "/maintenance" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 38, "<a class=\"nav-link active\" href=\"/maintenance\"><i class=\"fas fa-list me-2\"></i>Maintenance Queue</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "<a class=\"nav-link active\" href=\"/maintenance\"><i class=\"fas fa-list me-2\"></i>Job Queue</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "<a class=\"nav-link\" href=\"/maintenance\"><i class=\"fas fa-list me-2\"></i>Maintenance Queue</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "<a class=\"nav-link\" href=\"/maintenance\"><i class=\"fas fa-list me-2\"></i>Job Queue</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 40, "</li><li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "</li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if currentPath == "/maintenance/workers" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 41, "<a class=\"nav-link active\" href=\"/maintenance/workers\"><i class=\"fas fa-user-cog me-2\"></i>Maintenance Workers</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "<a class=\"nav-link active\" href=\"/maintenance/workers\"><i class=\"fas fa-user-cog me-2\"></i>Workers</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 42, "<a class=\"nav-link\" href=\"/maintenance/workers\"><i class=\"fas fa-user-cog me-2\"></i>Maintenance Workers</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "<a class=\"nav-link\" href=\"/maintenance/workers\"><i class=\"fas fa-user-cog me-2\"></i>Workers</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 43, "</li></ul></div></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-md-4\"><div class=\"pt-3\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "</li></ul></div></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-md-4\"><div class=\"pt-3\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -361,43 +481,43 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 44, "</div></main></div></div><!-- Footer --><footer class=\"footer mt-auto py-3 bg-light\"><div class=\"container-fluid text-center\"><small class=\"text-muted\">&copy; ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "</div></main></div></div><!-- Footer --><footer class=\"footer mt-auto py-3 bg-light\"><div class=\"container-fluid text-center\"><small class=\"text-muted\">&copy; ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var15 string
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
var templ_7745c5c3_Var25 string
templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 323, Col: 60}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 347, Col: 60}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 45, " SeaweedFS Admin v")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, " SeaweedFS Admin v")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var16 string
templ_7745c5c3_Var16, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
var templ_7745c5c3_Var26 string
templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 323, Col: 102}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 347, Col: 102}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var16))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 46, " ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, " ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if !strings.Contains(version.VERSION, "enterprise") {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 47, "<span class=\"mx-2\">•</span> <a href=\"https://seaweedfs.com\" target=\"_blank\" class=\"text-decoration-none\"><i class=\"fas fa-star me-1\"></i>Enterprise Version Available</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "<span class=\"mx-2\">•</span> <a href=\"https://seaweedfs.com\" target=\"_blank\" class=\"text-decoration-none\"><i class=\"fas fa-star me-1\"></i>Enterprise Version Available</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "</small></div></footer><!-- Bootstrap JS --><script src=\"/static/js/bootstrap.bundle.min.js\"></script><!-- Custom JS --><script src=\"/static/js/admin.js\"></script></body></html>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "</small></div></footer><!-- Bootstrap JS --><script src=\"/static/js/bootstrap.bundle.min.js\"></script><!-- Custom JS --><script src=\"/static/js/admin.js\"></script></body></html>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -421,61 +541,61 @@ func LoginForm(c *gin.Context, title string, errorMessage string) templ.Componen
}()
}
ctx = templ.InitializeContext(ctx)
templ_7745c5c3_Var17 := templ.GetChildren(ctx)
if templ_7745c5c3_Var17 == nil {
templ_7745c5c3_Var17 = templ.NopComponent
templ_7745c5c3_Var27 := templ.GetChildren(ctx)
if templ_7745c5c3_Var27 == nil {
templ_7745c5c3_Var27 = templ.NopComponent
}
ctx = templ.ClearChildren(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 49, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var18 string
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(title)
var templ_7745c5c3_Var28 string
templ_7745c5c3_Var28, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 347, Col: 17}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 371, Col: 17}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var28))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, " - Login</title><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"></head><body class=\"bg-light\"><div class=\"container\"><div class=\"row justify-content-center min-vh-100 align-items-center\"><div class=\"col-md-6 col-lg-4\"><div class=\"card shadow\"><div class=\"card-body p-5\"><div class=\"text-center mb-4\"><i class=\"fas fa-server fa-3x text-primary mb-3\"></i><h4 class=\"card-title\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, " - Login</title><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"></head><body class=\"bg-light\"><div class=\"container\"><div class=\"row justify-content-center min-vh-100 align-items-center\"><div class=\"col-md-6 col-lg-4\"><div class=\"card shadow\"><div class=\"card-body p-5\"><div class=\"text-center mb-4\"><i class=\"fas fa-server fa-3x text-primary mb-3\"></i><h4 class=\"card-title\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var19 string
templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(title)
var templ_7745c5c3_Var29 string
templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 361, Col: 57}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 385, Col: 57}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 51, "</h4><p class=\"text-muted\">Please sign in to continue</p></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, "</h4><p class=\"text-muted\">Please sign in to continue</p></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if errorMessage != "" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 52, "<div class=\"alert alert-danger\" role=\"alert\"><i class=\"fas fa-exclamation-triangle me-2\"></i> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "<div class=\"alert alert-danger\" role=\"alert\"><i class=\"fas fa-exclamation-triangle me-2\"></i> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var20 string
templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
var templ_7745c5c3_Var30 string
templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 368, Col: 45}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 392, Col: 45}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 53, "</div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "</div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "<form method=\"POST\" action=\"/login\"><div class=\"mb-3\"><label for=\"username\" class=\"form-label\">Username</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"fas fa-user\"></i></span> <input type=\"text\" class=\"form-control\" id=\"username\" name=\"username\" required></div></div><div class=\"mb-4\"><label for=\"password\" class=\"form-label\">Password</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"fas fa-lock\"></i></span> <input type=\"password\" class=\"form-control\" id=\"password\" name=\"password\" required></div></div><button type=\"submit\" class=\"btn btn-primary w-100\"><i class=\"fas fa-sign-in-alt me-2\"></i>Sign In</button></form></div></div></div></div></div><script src=\"/static/js/bootstrap.bundle.min.js\"></script></body></html>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "<form method=\"POST\" action=\"/login\"><div class=\"mb-3\"><label for=\"username\" class=\"form-label\">Username</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"fas fa-user\"></i></span> <input type=\"text\" class=\"form-control\" id=\"username\" name=\"username\" required></div></div><div class=\"mb-4\"><label for=\"password\" class=\"form-label\">Password</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"fas fa-lock\"></i></span> <input type=\"password\" class=\"form-control\" id=\"password\" name=\"password\" required></div></div><button type=\"submit\" class=\"btn btn-primary w-100\"><i class=\"fas fa-sign-in-alt me-2\"></i>Sign In</button></form></div></div></div></div></div><script src=\"/static/js/bootstrap.bundle.min.js\"></script></body></html>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
weed/command/admin.go
+42 -10
View File
@@ -33,13 +33,15 @@ var (
)
type AdminOptions struct {
port *int
grpcPort *int
master *string
masters *string // deprecated, for backward compatibility
adminUser *string
adminPassword *string
dataDir *string
port *int
grpcPort *int
master *string
masters *string // deprecated, for backward compatibility
adminUser *string
adminPassword *string
readOnlyUser *string
readOnlyPassword *string
dataDir *string
}
func init() {
@@ -52,6 +54,8 @@ func init() {
a.adminUser = cmdAdmin.Flag.String("adminUser", "admin", "admin interface username")
a.adminPassword = cmdAdmin.Flag.String("adminPassword", "", "admin interface password (if empty, auth is disabled)")
a.readOnlyUser = cmdAdmin.Flag.String("readOnlyUser", "", "read-only user username (optional, for view-only access)")
a.readOnlyPassword = cmdAdmin.Flag.String("readOnlyPassword", "", "read-only user password (optional, for view-only access; requires adminPassword to be set)")
}
var cmdAdmin = &Command{
@@ -84,7 +88,11 @@ var cmdAdmin = &Command{
Authentication:
- If adminPassword is not set, the admin interface runs without authentication
- If adminPassword is set, users must login with adminUser/adminPassword
- If adminPassword is set, users must login with adminUser/adminPassword (full access)
- Optional read-only access: set readOnlyUser and readOnlyPassword for view-only access
- Read-only users can view cluster status and configurations but cannot make changes
- IMPORTANT: When read-only credentials are configured, adminPassword MUST also be set
- This ensures an admin account exists to manage and authorize read-only access
- Sessions are secured with auto-generated session keys
Security Configuration:
@@ -139,6 +147,26 @@ func runAdmin(cmd *Command, args []string) bool {
return false
}
// Security validation: prevent empty username when password is set
if *a.adminPassword != "" && *a.adminUser == "" {
fmt.Println("Error: -adminUser cannot be empty when -adminPassword is set")
return false
}
if *a.readOnlyPassword != "" && *a.readOnlyUser == "" {
fmt.Println("Error: -readOnlyUser is required when -readOnlyPassword is set")
return false
}
// Security validation: prevent username conflicts between admin and read-only users
if *a.adminUser != "" && *a.readOnlyUser != "" && *a.adminUser == *a.readOnlyUser {
fmt.Println("Error: -adminUser and -readOnlyUser must be different when both are configured")
return false
}
// Security validation: admin password is required for read-only user
if *a.readOnlyPassword != "" && *a.adminPassword == "" {
fmt.Println("Error: -adminPassword must be set when -readOnlyPassword is configured")
return false
}
// Set default gRPC port if not specified
if *a.grpcPort == 0 {
*a.grpcPort = *a.port + 10000
@@ -160,7 +188,10 @@ func runAdmin(cmd *Command, args []string) bool {
fmt.Printf("Data Directory: Not specified (configuration will be in-memory only)\n")
}
if *a.adminPassword != "" {
fmt.Printf("Authentication: Enabled (user: %s)\n", *a.adminUser)
fmt.Printf("Authentication: Enabled (admin user: %s)\n", *a.adminUser)
if *a.readOnlyPassword != "" {
fmt.Printf("Read-only access: Enabled (read-only user: %s)\n", *a.readOnlyUser)
}
} else {
fmt.Printf("Authentication: Disabled\n")
}
@@ -274,8 +305,9 @@ func startAdminServer(ctx context.Context, options AdminOptions) error {
}()
// Create handlers and setup routes
authRequired := *options.adminPassword != ""
adminHandlers := handlers.NewAdminHandlers(adminServer)
adminHandlers.SetupRoutes(r, *options.adminPassword != "", *options.adminUser, *options.adminPassword)
adminHandlers.SetupRoutes(r, authRequired, *options.adminUser, *options.adminPassword, *options.readOnlyUser, *options.readOnlyPassword)
// Server configuration
addr := fmt.Sprintf(":%d", *options.port)
weed/command/filer_backup.go
+13 -5
View File
@@ -3,6 +3,10 @@ package command
import (
"errors"
"fmt"
"regexp"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -11,9 +15,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/http"
"google.golang.org/grpc"
"regexp"
"strings"
"time"
)
type FilerBackupOptions struct {
@@ -82,8 +83,8 @@ func runFilerBackup(cmd *Command, args []string) bool {
time.Sleep(1747 * time.Millisecond)
}
}
return true
// Unreachable: satisfies bool return type signature for daemon function
return false
}
const (
@@ -145,10 +146,17 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
if err == nil {
return nil
}
// ignore HTTP 404 from remote reads
if errors.Is(err, http.ErrNotFound) {
glog.V(0).Infof("got 404 error, ignore it: %s", err.Error())
return nil
}
// also ignore missing volume/lookup errors coming from LookupFileId or vid map
errStr := err.Error()
if strings.Contains(errStr, "LookupFileId") || (strings.Contains(errStr, "volume id") && strings.Contains(errStr, "not found")) {
glog.V(0).Infof("got missing-volume error, ignore it: %s", errStr)
return nil
}
return err
}
} else {
weed/command/filer_meta_backup.go
+82 -20
View File
@@ -3,13 +3,14 @@ package command
import (
"context"
"fmt"
"reflect"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/spf13/viper"
"google.golang.org/grpc"
"reflect"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -25,9 +26,12 @@ type FilerMetaBackupOptions struct {
grpcDialOption grpc.DialOption
filerAddress *string
filerDirectory *string
includePrefixes *string
excludePrefixes *string
restart *bool
backupFilerConfig *string
pathFilter *util.PathPrefixFilter
store filer.FilerStore
clientId int32
clientEpoch int32
@@ -37,20 +41,25 @@ func init() {
cmdFilerMetaBackup.Run = runFilerMetaBackup // break init cycle
metaBackup.filerAddress = cmdFilerMetaBackup.Flag.String("filer", "localhost:8888", "filer hostname:port")
metaBackup.filerDirectory = cmdFilerMetaBackup.Flag.String("filerDir", "/", "a folder on the filer")
metaBackup.includePrefixes = cmdFilerMetaBackup.Flag.String("includePrefixes", "", "comma-separated path prefixes to include in backup (if set, only these paths are backed up)")
metaBackup.excludePrefixes = cmdFilerMetaBackup.Flag.String("excludePrefixes", "", "comma-separated path prefixes to exclude from backup")
metaBackup.restart = cmdFilerMetaBackup.Flag.Bool("restart", false, "copy the full metadata before async incremental backup")
metaBackup.backupFilerConfig = cmdFilerMetaBackup.Flag.String("config", "", "path to filer.toml specifying backup filer store")
metaBackup.clientId = util.RandomInt32()
}
var cmdFilerMetaBackup = &Command{
UsageLine: "filer.meta.backup [-filer=localhost:8888] [-filerDir=/] [-restart] -config=/path/to/backup_filer.toml",
UsageLine: "filer.meta.backup [-filer=localhost:8888] [-filerDir=/] [-includePrefixes=...] [-excludePrefixes=...] [-restart] -config=/path/to/backup_filer.toml",
Short: "continuously backup filer meta data changes to anther filer store specified in a backup_filer.toml",
Long: `continuously backup filer meta data changes.
Long: `continuously backup filer meta data changes.
The backup writes to another filer store specified in a backup_filer.toml.
weed filer.meta.backup -config=/path/to/backup_filer.toml -filer="localhost:8888"
weed filer.meta.backup -config=/path/to/backup_filer.toml -filer="localhost:8888" -restart
The -includePrefixes and -excludePrefixes flags accept comma-separated path prefixes.
Paths must be absolute (start with '/'). Matching is at directory boundaries.
When both match, the deeper prefix wins.
`,
}
@@ -74,6 +83,23 @@ func runFilerMetaBackup(cmd *Command, args []string) bool {
return true
}
// Initialize path filter
metaBackup.pathFilter = util.NewPathPrefixFilter(
*metaBackup.includePrefixes,
*metaBackup.excludePrefixes,
func(format string, args ...interface{}) {
glog.Warningf(format, args...)
},
)
if metaBackup.pathFilter.HasFilters() {
if len(metaBackup.pathFilter.GetIncludePrefixes()) > 0 {
glog.V(0).Infof("including prefixes: %v", metaBackup.pathFilter.GetIncludePrefixes())
}
if len(metaBackup.pathFilter.GetExcludePrefixes()) > 0 {
glog.V(0).Infof("excluding prefixes: %v", metaBackup.pathFilter.GetExcludePrefixes())
}
}
missingPreviousBackup := false
_, err := metaBackup.getOffset()
if err != nil {
@@ -100,8 +126,8 @@ func runFilerMetaBackup(cmd *Command, args []string) bool {
time.Sleep(1747 * time.Millisecond)
}
}
return true
// Unreachable: satisfies bool return type signature for daemon function
return false
}
func (metaBackup *FilerMetaBackupOptions) initStore(v *viper.Viper) error {
@@ -126,12 +152,22 @@ func (metaBackup *FilerMetaBackupOptions) initStore(v *viper.Viper) error {
return nil
}
// shouldInclude checks if the given path should be included in backup
// based on the configured include/exclude path prefixes.
func (metaBackup *FilerMetaBackupOptions) shouldInclude(fullpath string) bool {
return metaBackup.pathFilter.ShouldInclude(fullpath)
}
func (metaBackup *FilerMetaBackupOptions) traverseMetadata() (err error) {
var saveErr error
traverseErr := filer_pb.TraverseBfs(metaBackup, util.FullPath(*metaBackup.filerDirectory), func(parentPath util.FullPath, entry *filer_pb.Entry) {
fullpath := string(parentPath.Child(entry.Name))
if !metaBackup.shouldInclude(fullpath) {
return
}
println("+", parentPath.Child(entry.Name))
println("+", fullpath)
if err := metaBackup.store.InsertEntry(context.Background(), filer.FromPbEntry(string(parentPath), entry)); err != nil {
saveErr = fmt.Errorf("insert entry error: %w\n", err)
return
@@ -166,28 +202,54 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
if filer_pb.IsEmpty(resp) {
return nil
} else if filer_pb.IsCreate(resp) {
println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
}
// Compute exclusion for both old and new paths
var oldPathExcluded, newPathExcluded bool
var oldPath, newPath string
if message.OldEntry != nil {
oldPath = string(util.FullPath(resp.Directory).Child(message.OldEntry.Name))
oldPathExcluded = !metaBackup.shouldInclude(oldPath)
}
if message.NewEntry != nil {
newPath = string(util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
newPathExcluded = !metaBackup.shouldInclude(newPath)
}
if filer_pb.IsCreate(resp) {
if newPathExcluded {
return nil
}
println("+", newPath)
entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry)
return store.InsertEntry(ctx, entry)
} else if filer_pb.IsDelete(resp) {
println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name))
if oldPathExcluded {
return nil
}
println("-", oldPath)
return store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name))
} else if filer_pb.IsUpdate(resp) {
println("~", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
if newPathExcluded {
return nil
}
println("~", newPath)
entry := filer.FromPbEntry(message.NewParentPath, message.NewEntry)
return store.UpdateEntry(ctx, entry)
} else {
// renaming
println("-", util.FullPath(resp.Directory).Child(message.OldEntry.Name))
if err := store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)); err != nil {
return err
// renaming - handle all four combinations
if !oldPathExcluded {
println("-", oldPath)
if err := store.DeleteEntry(ctx, util.FullPath(resp.Directory).Child(message.OldEntry.Name)); err != nil {
return err
}
}
println("+", util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
return store.InsertEntry(ctx, filer.FromPbEntry(message.NewParentPath, message.NewEntry))
if !newPathExcluded {
println("+", newPath)
return store.InsertEntry(ctx, filer.FromPbEntry(message.NewParentPath, message.NewEntry))
}
return nil
}
return nil
}
processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
weed/command/filer_sync.go
-2
View File
@@ -262,8 +262,6 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
}
select {}
return true
}
// initOffsetFromTsMs Initialize offset
weed/command/fuse.go
+4
View File
@@ -28,3 +28,7 @@ var cmdFuse = &Command{
To check valid options look "weed mount --help"
`,
}
func GetFuseCommandName() string {
return cmdFuse.Name()
}
weed/command/fuse_std.go
+7
View File
@@ -12,6 +12,9 @@ import (
"strings"
"syscall"
"time"
"github.com/seaweedfs/seaweedfs/weed/util"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
type parameter struct {
@@ -219,6 +222,8 @@ func runFuse(cmd *Command, args []string) bool {
}
case "fusermount.path":
fusermountPath = parameter.value
case "config_dir":
util.ConfigurationFileDirectory.Set(parameter.value)
default:
t := parameter.name
if parameter.value != "true" {
@@ -228,6 +233,8 @@ func runFuse(cmd *Command, args []string) bool {
}
}
util_http.InitGlobalHttpClient()
// the master start the child, release it then finish himself
if masterProcess {
arg0, err := os.Executable()
weed/command/mini.go
+492 -16
View File
@@ -3,6 +3,7 @@ package command
import (
"context"
"fmt"
"math/bits"
"net"
"net/http"
"os"
@@ -17,6 +18,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/security"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/worker"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
@@ -36,8 +38,13 @@ type MiniOptions struct {
}
const (
miniVolumeMaxDataVolumeCounts = "0" // auto-configured based on free disk space
miniVolumeMinFreeSpace = "1" // 1% minimum free space
bytesPerMB = 1024 * 1024 // Bytes per MB
miniVolumeMaxDataVolumeCounts = "0" // auto-configured based on free disk space
miniVolumeMinFreeSpace = "1" // 1% minimum free space
minVolumeSizeMB = 64 // Minimum volume size in MB
defaultMiniVolumeSizeMB = 128 // Default volume size for mini mode
maxVolumeSizeMB = 1024 // Maximum volume size in MB (1GB)
GrpcPortOffset = 10000 // Offset used to calculate gRPC port from HTTP port
)
var (
@@ -48,6 +55,8 @@ var (
miniWebDavOptions WebDavOption
miniAdminOptions AdminOptions
createdInitialIAM bool // Track if initial IAM config was created from env vars
// Track which port flags were explicitly passed on CLI before config file is applied
explicitPortFlags map[string]bool
)
func init() {
@@ -63,7 +72,7 @@ This command starts all components in one process (master, volume, filer,
S3 gateway, WebDAV gateway, and Admin UI).
All settings are optimized for small/dev use cases:
- Volume size limit: 128MB (small files)
- Volume size limit: auto configured based on disk space (64MB-1024MB)
- Volume max: 0 (auto-configured based on free disk space)
- Pre-stop seconds: 1 (faster shutdown)
- Master peers: none (single master mode)
@@ -111,6 +120,15 @@ var (
miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
)
// getBindIp determines the bind IP address based on miniIp and miniBindIp flags
// Returns miniBindIp if set (non-empty), otherwise returns miniIp
func getBindIp() string {
if *miniBindIp != "" {
return *miniBindIp
}
return *miniIp
}
// initMiniCommonFlags initializes common mini flags
func initMiniCommonFlags() {
miniOptions.cpuprofile = cmdMini.Flag.String("cpuprofile", "", "cpu profile output file")
@@ -125,7 +143,7 @@ func initMiniMasterFlags() {
miniMasterOptions.portGrpc = cmdMini.Flag.Int("master.port.grpc", 0, "master server grpc listen port")
miniMasterOptions.metaFolder = cmdMini.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
miniMasterOptions.peers = cmdMini.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list (default: none for single master)")
miniMasterOptions.volumeSizeLimitMB = cmdMini.Flag.Uint("master.volumeSizeLimitMB", 128, "Master stops directing writes to oversized volumes (default: 128MB for mini)")
miniMasterOptions.volumeSizeLimitMB = cmdMini.Flag.Uint("master.volumeSizeLimitMB", defaultMiniVolumeSizeMB, "Master stops directing writes to oversized volumes (default: 128MB for mini)")
miniMasterOptions.volumePreallocate = cmdMini.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
miniMasterOptions.maxParallelVacuumPerServer = cmdMini.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server")
miniMasterOptions.defaultReplication = cmdMini.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
@@ -211,12 +229,14 @@ func initMiniS3Flags() {
miniS3Options.concurrentFileUploadLimit = cmdMini.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads")
miniS3Options.enableIam = cmdMini.Flag.Bool("s3.iam", true, "enable embedded IAM API on the same port")
miniS3Options.dataCenter = cmdMini.Flag.String("s3.dataCenter", "", "prefer to read and write to volumes in this data center")
miniS3Options.cipher = cmdMini.Flag.Bool("s3.encryptVolumeData", false, "encrypt data on volume servers for S3 uploads")
miniS3Options.config = miniS3Config
miniS3Options.iamConfig = miniIamConfig
miniS3Options.auditLogConfig = cmdMini.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
miniS3Options.allowDeleteBucketNotEmpty = miniS3AllowDeleteBucketNotEmpty
miniS3Options.debug = cmdMini.Flag.Bool("s3.debug", false, "serves runtime profiling data via pprof")
miniS3Options.debugPort = cmdMini.Flag.Int("s3.debug.port", 6060, "http port for debugging")
// In mini mode, S3 uses the shared debug server started at line 681, not its own separate debug server
miniS3Options.debug = new(bool) // explicitly false
miniS3Options.debugPort = cmdMini.Flag.Int("s3.debug.port", 6060, "http port for debugging (unused in mini mode)")
}
// initMiniWebDAVFlags initializes WebDAV server flag options
@@ -236,11 +256,13 @@ func initMiniWebDAVFlags() {
// initMiniAdminFlags initializes Admin server flag options
func initMiniAdminFlags() {
miniAdminOptions.port = cmdMini.Flag.Int("admin.port", 23646, "admin server http listen port")
miniAdminOptions.grpcPort = cmdMini.Flag.Int("admin.port.grpc", 0, "admin server grpc listen port (default: admin http port + 10000)")
miniAdminOptions.grpcPort = cmdMini.Flag.Int("admin.port.grpc", 0, "admin server grpc listen port (default: admin http port + GrpcPortOffset)")
miniAdminOptions.master = cmdMini.Flag.String("admin.master", "", "master server address (automatically set)")
miniAdminOptions.dataDir = cmdMini.Flag.String("admin.dataDir", "", "directory to store admin configuration and data files")
miniAdminOptions.adminUser = cmdMini.Flag.String("admin.user", "admin", "admin interface username")
miniAdminOptions.adminPassword = cmdMini.Flag.String("admin.password", "", "admin interface password (if empty, auth is disabled)")
miniAdminOptions.readOnlyUser = cmdMini.Flag.String("admin.readOnlyUser", "", "read-only user username (optional, for view-only access)")
miniAdminOptions.readOnlyPassword = cmdMini.Flag.String("admin.readOnlyPassword", "", "read-only user password (optional, for view-only access; requires admin.password to be set)")
}
func init() {
@@ -256,8 +278,409 @@ func init() {
initMiniAdminFlags()
}
// calculateOptimalVolumeSizeMB calculates optimal volume size based on total disk capacity.
//
// Algorithm:
// 1. Read total disk capacity using the OS-independent stats.NewDiskStatus()
// 2. Convert capacity from bytes to MB, then divide by 100
// 3. Round up to nearest power of 2 (64MB, 128MB, 256MB, 512MB, 1024MB, etc.)
// 4. Clamp the result to range [64MB, 1024MB]
//
// Examples (GB→MB conversion, divide by 100, round to next power-of-2, clamp [64,1024]):
// - 10GB disk → 10240MB / 100 = 102.4MB → rounds to 128MB
// - 100GB disk → 102400MB / 100 = 1024MB → rounds to 1024MB
// - 500GB disk → 512000MB / 100 = 5120MB → rounds to 8192MB → capped to 1024MB
// - 1TB disk → 1048576MB / 100 = 10485.76MB → capped to 1024MB (maximum)
// - 6.4TB disk → 6553600MB / 100 = 65536MB → capped to 1024MB (maximum)
// - 12.8TB disk → 13107200MB / 100 = 131072MB → capped to 1024MB (maximum)
func calculateOptimalVolumeSizeMB(dataFolder string) uint {
// Get disk status for the data folder using OS-independent function
diskStatus := stats_collect.NewDiskStatus(dataFolder)
if diskStatus == nil || diskStatus.All == 0 {
glog.Warningf("Could not determine disk size, using default %dMB", defaultMiniVolumeSizeMB)
return defaultMiniVolumeSizeMB
}
// Calculate optimal size: total disk capacity / 100 for stability
// Using total capacity (All) instead of free space ensures consistent volume size
// regardless of current disk usage. diskStatus.All is in bytes, convert to MB
totalCapacityMB := diskStatus.All / bytesPerMB
initialOptimalMB := uint(totalCapacityMB / 100)
optimalMB := initialOptimalMB
// Round up to nearest power of 2: 64MB, 128MB, 256MB, 512MB, etc.
// Minimum is 64MB, maximum is 1024MB (1GB)
if optimalMB == 0 {
// If the computed optimal size is 0, start from the minimum volume size
optimalMB = minVolumeSizeMB
} else {
// Round up to the nearest power of 2
optimalMB = 1 << bits.Len(optimalMB-1)
}
// Apply the minimum and maximum constraints
if optimalMB < minVolumeSizeMB {
optimalMB = minVolumeSizeMB
} else if optimalMB > maxVolumeSizeMB {
optimalMB = maxVolumeSizeMB
}
glog.Infof("Optimal volume size: %dMB (total disk capacity: %dMB, capacity/100 before rounding: %dMB, rounded to nearest power of 2, clamped to [%d,%d]MB)",
optimalMB, totalCapacityMB, initialOptimalMB, minVolumeSizeMB, maxVolumeSizeMB)
return optimalMB
}
// isFlagPassed checks if a specific flag was passed on the command line
func isFlagPassed(name string) bool {
found := false
cmdMini.Flag.Visit(func(f *flag.Flag) {
if f.Name == name {
found = true
}
})
return found
}
// isPortOpenOnIP checks if a port is available for binding on a specific IP address
func isPortOpenOnIP(ip string, port int) bool {
listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", ip, port))
if err != nil {
return false
}
listener.Close()
return true
}
// isPortAvailable checks if a port is available on any interface
// This is more comprehensive than checking a single IP
func isPortAvailable(port int) bool {
// Try to listen on all interfaces (0.0.0.0)
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
return false
}
listener.Close()
return true
}
// findAvailablePortOnIP finds the next available port on a specific IP starting from the given port
// It skips any ports that are in the reservedPorts map (for gRPC port collision avoidance)
// It returns the first available port found within maxAttempts, or 0 if none found
func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPorts map[int]bool) int {
for i := 0; i < maxAttempts; i++ {
port := startPort + i
// Skip ports reserved for gRPC calculation
if reservedPorts[port] {
continue
}
// Check on both the specific IP and on all interfaces for maximum reliability
if isPortOpenOnIP(ip, port) && isPortAvailable(port) {
return port
}
}
// If no port found, return 0 to indicate failure
return 0
}
// ensurePortAvailableOnIP ensures a port pointer points to an available port on a specific IP
// If the port is not available, it finds the next available port and updates the pointer
// The reservedPorts map contains ports that should not be allocated (for gRPC collision avoidance)
func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reservedPorts map[int]bool, flagName string) error {
if portPtr == nil {
return nil
}
original := *portPtr
// Check if this port was explicitly specified by the user (from CLI, before config file was applied)
isExplicitPort := explicitPortFlags[flagName]
// Skip if this port is reserved for gRPC calculation
if reservedPorts[original] {
if isExplicitPort {
return fmt.Errorf("port %d for %s (specified by flag %s) is reserved for gRPC calculation and cannot be used", original, serviceName, flagName)
}
glog.Warningf("Port %d for %s is reserved for gRPC calculation, finding alternative...", original, serviceName)
newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts)
if newPort == 0 {
glog.Errorf("Could not find available port for %s starting from %d, will use original %d and fail on binding", serviceName, original+1, original)
} else {
glog.Infof("Port %d for %s is available, using it instead of %d", newPort, serviceName, original)
*portPtr = newPort
}
return nil
}
// Check on both the specific IP and on all interfaces (0.0.0.0) for maximum reliability
if !isPortOpenOnIP(ip, original) || !isPortAvailable(original) {
// If explicitly specified, fail immediately with the originally requested port
if isExplicitPort {
return fmt.Errorf("port %d for %s (specified by flag %s) is not available on %s and cannot be used", original, serviceName, flagName, ip)
}
// For default ports, try to find an alternative
glog.Warningf("Port %d for %s is not available on %s, finding alternative port...", original, serviceName, ip)
newPort := findAvailablePortOnIP(ip, original+1, 100, reservedPorts)
if newPort == 0 {
glog.Errorf("Could not find available port for %s starting from %d, will use original %d and fail on binding", serviceName, original+1, original)
} else {
glog.Infof("Port %d for %s is available, using it instead of %d", newPort, serviceName, original)
*portPtr = newPort
}
} else {
glog.V(1).Infof("Port %d for %s is available on %s", original, serviceName, ip)
}
return nil
}
// ensureAllPortsAvailableOnIP ensures all mini service ports are available on a specific IP
// Returns an error if an explicitly specified port is unavailable.
// This should be called before starting any services
func ensureAllPortsAvailableOnIP(bindIp string) error {
portConfigs := []struct {
port *int
name string
flagName string
grpcPtr *int
}{
{miniMasterOptions.port, "Master", "master.port", miniMasterOptions.portGrpc},
{miniFilerOptions.port, "Filer", "filer.port", miniFilerOptions.portGrpc},
{miniOptions.v.port, "Volume", "volume.port", miniOptions.v.portGrpc},
{miniS3Options.port, "S3", "s3.port", miniS3Options.portGrpc},
{miniWebDavOptions.port, "WebDAV", "webdav.port", nil},
{miniAdminOptions.port, "Admin", "admin.port", miniAdminOptions.grpcPort},
}
// First, reserve all gRPC ports that will be calculated to prevent HTTP port allocation from using them
// This prevents collisions like: HTTP port moves to X, then gRPC port is calculated as Y where Y == X
reservedPorts := make(map[int]bool)
for _, config := range portConfigs {
if config.grpcPtr != nil && *config.grpcPtr == 0 {
// This gRPC port will be calculated as httpPort + GrpcPortOffset
calculatedGrpcPort := *config.port + GrpcPortOffset
reservedPorts[calculatedGrpcPort] = true
}
}
// Check all HTTP ports sequentially to avoid race conditions
// Each port check and allocation must complete before the next one starts
// to prevent multiple goroutines from claiming the same available port
// Also avoid allocating ports that are reserved for gRPC calculation
for _, config := range portConfigs {
original := *config.port
if err := ensurePortAvailableOnIP(config.port, config.name, bindIp, reservedPorts, config.flagName); err != nil {
return err
}
// If port was changed, update the reserved gRPC ports mapping
if *config.port != original && config.grpcPtr != nil && *config.grpcPtr == 0 {
delete(reservedPorts, original+GrpcPortOffset)
reservedPorts[*config.port+GrpcPortOffset] = true
}
}
// Initialize all gRPC ports before services start
// This ensures they won't be recalculated and cause conflicts
// All gRPC port handling (calculation, validation, and assignment) is performed exclusively in initializeGrpcPortsOnIP
initializeGrpcPortsOnIP(bindIp)
// Log the final port configuration
glog.Infof("Final port configuration - Master: %d, Filer: %d, Volume: %d, S3: %d, WebDAV: %d, Admin: %d",
*miniMasterOptions.port, *miniFilerOptions.port, *miniOptions.v.port,
*miniS3Options.port, *miniWebDavOptions.port, *miniAdminOptions.port)
// Log gRPC ports too (now finalized)
glog.Infof("gRPC port configuration - Master: %d, Filer: %d, Volume: %d, S3: %d, Admin: %d",
*miniMasterOptions.portGrpc, *miniFilerOptions.portGrpc, *miniOptions.v.portGrpc,
*miniS3Options.portGrpc, *miniAdminOptions.grpcPort)
return nil
}
// initializeGrpcPortsOnIP initializes all gRPC ports based on their HTTP ports on a specific IP
// If a gRPC port is 0, it will be set to httpPort + GrpcPortOffset
// This must be called after HTTP ports are finalized and before services start
func initializeGrpcPortsOnIP(bindIp string) {
// Track gRPC ports allocated during this function to prevent collisions between services
// when multiple services need fallback port allocation
allocatedGrpcPorts := make(map[int]bool)
grpcConfigs := []struct {
httpPort *int
grpcPort *int
name string
}{
{miniMasterOptions.port, miniMasterOptions.portGrpc, "Master"},
{miniFilerOptions.port, miniFilerOptions.portGrpc, "Filer"},
{miniOptions.v.port, miniOptions.v.portGrpc, "Volume"},
{miniS3Options.port, miniS3Options.portGrpc, "S3"},
{miniAdminOptions.port, miniAdminOptions.grpcPort, "Admin"},
}
for _, config := range grpcConfigs {
if config.grpcPort == nil {
continue
}
// If gRPC port is 0, calculate it from HTTP port
if *config.grpcPort == 0 {
*config.grpcPort = *config.httpPort + GrpcPortOffset
}
// Verify the gRPC port is available (whether calculated or explicitly set)
// Check on both specific IP and all interfaces, and check against already allocated ports
if !isPortOpenOnIP(bindIp, *config.grpcPort) || !isPortAvailable(*config.grpcPort) || allocatedGrpcPorts[*config.grpcPort] {
glog.Warningf("gRPC port %d for %s is not available, finding alternative...", *config.grpcPort, config.name)
originalPort := *config.grpcPort
newPort := findAvailablePortOnIP(bindIp, originalPort+1, 100, allocatedGrpcPorts)
if newPort == 0 {
glog.Errorf("Could not find available gRPC port for %s starting from %d, will use %d and fail on binding", config.name, originalPort+1, originalPort)
} else {
glog.Infof("gRPC port %d for %s is available, using it instead of %d", newPort, config.name, originalPort)
*config.grpcPort = newPort
}
}
allocatedGrpcPorts[*config.grpcPort] = true
glog.V(1).Infof("%s gRPC port set to %d", config.name, *config.grpcPort)
}
}
// loadMiniConfigurationFile reads the mini.options file and returns parsed options
// File format: one option per line, without leading dash (e.g., "ip=127.0.0.1")
func loadMiniConfigurationFile(dataFolder string) (map[string]string, error) {
configFile := filepath.Join(util.ResolvePath(util.StringSplit(dataFolder, ",")[0]), "mini.options")
options := make(map[string]string)
// Check if file exists
data, err := os.ReadFile(configFile)
if err != nil {
if os.IsNotExist(err) {
// File doesn't exist - this is OK, return empty options
return options, nil
}
glog.Warningf("Failed to read configuration file %s: %v", configFile, err)
return options, err
}
// Parse the file line by line
lines := strings.Split(string(data), "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
// Skip empty lines and comments
if len(line) == 0 || strings.HasPrefix(line, "#") {
continue
}
// Remove leading dash if present
if strings.HasPrefix(line, "-") {
line = line[1:]
}
// Parse key=value
parts := strings.SplitN(line, "=", 2)
if len(parts) == 2 {
key := strings.TrimSpace(parts[0])
value := strings.TrimSpace(parts[1])
// Remove quotes if present
if (strings.HasPrefix(value, "\"") && strings.HasSuffix(value, "\"")) ||
(strings.HasPrefix(value, "'") && strings.HasSuffix(value, "'")) {
value = value[1 : len(value)-1]
}
options[key] = value
}
}
glog.Infof("Loaded %d options from configuration file %s", len(options), configFile)
return options, nil
}
// applyConfigFileOptions sets command-line flags from loaded configuration file
func applyConfigFileOptions(options map[string]string) {
for key, value := range options {
// Skip port flags that were explicitly passed on CLI
if explicitPortFlags[key] {
glog.V(2).Infof("Skipping config file option %s=%s (explicitly specified on command line)", key, value)
continue
}
// Set the flag value if it hasn't been explicitly set on command line
flag := cmdMini.Flag.Lookup(key)
if flag != nil {
// Only set if not already set (by command line)
if flag.Value.String() == flag.DefValue {
flag.Value.Set(value)
glog.V(2).Infof("Applied config file option: %s=%s", key, value)
}
}
}
}
// saveMiniConfiguration saves the current mini configuration to a file
// The file format uses option=value format without leading dashes
func saveMiniConfiguration(dataFolder string) error {
configDir := util.ResolvePath(util.StringSplit(dataFolder, ",")[0])
if err := os.MkdirAll(configDir, 0755); err != nil {
glog.Warningf("Failed to create config directory %s: %v", configDir, err)
return err
}
configFile := filepath.Join(configDir, "mini.options")
var sb strings.Builder
sb.WriteString("#!/bin/bash\n")
sb.WriteString("# Mini server configuration\n")
sb.WriteString("# Format: option=value (no leading dash)\n")
sb.WriteString("# This file is loaded on startup if it exists\n\n")
// Collect all flags that were explicitly passed (except "dir")
cmdMini.Flag.Visit(func(f *flag.Flag) {
// Skip the "dir" option - it's environment-specific
if f.Name == "dir" {
return
}
value := f.Value.String()
// Quote the value if it contains spaces
if strings.Contains(value, " ") {
sb.WriteString(fmt.Sprintf("%s=\"%s\"\n", f.Name, value))
} else {
sb.WriteString(fmt.Sprintf("%s=%s\n", f.Name, value))
}
})
// Add auto-calculated volume size if it was computed
if !isFlagPassed("master.volumeSizeLimitMB") && miniMasterOptions.volumeSizeLimitMB != nil {
sb.WriteString(fmt.Sprintf("\n# Auto-calculated volume size based on total disk capacity\n"))
sb.WriteString(fmt.Sprintf("# Delete this line to force recalculation on next startup\n"))
sb.WriteString(fmt.Sprintf("master.volumeSizeLimitMB=%d\n", *miniMasterOptions.volumeSizeLimitMB))
}
if err := os.WriteFile(configFile, []byte(sb.String()), 0644); err != nil {
glog.Warningf("Failed to save configuration to %s: %v", configFile, err)
return err
}
glog.Infof("Mini configuration saved to %s", configFile)
return nil
}
func runMini(cmd *Command, args []string) bool {
// Capture which port flags were explicitly passed on CLI BEFORE config file is applied
// This is necessary to distinguish user-specified ports from defaults or config file options
explicitPortFlags = make(map[string]bool)
portFlagNames := []string{"master.port", "filer.port", "volume.port", "s3.port", "webdav.port", "admin.port"}
for _, flagName := range portFlagNames {
explicitPortFlags[flagName] = isFlagPassed(flagName)
}
// Load configuration from file if it exists
configOptions, err := loadMiniConfigurationFile(*miniDataFolders)
if err != nil {
glog.Warningf("Error loading configuration file: %v", err)
}
// Apply loaded options to flags (CLI flags will override these)
applyConfigFileOptions(configOptions)
if *miniOptions.debug {
grace.StartDebugServer(*miniOptions.debugPort)
}
@@ -267,6 +690,15 @@ func runMini(cmd *Command, args []string) bool {
grace.SetupProfiling(*miniOptions.cpuprofile, *miniOptions.memprofile)
// Determine bind IP
bindIp := getBindIp()
// Ensure all ports are available, find alternatives if needed
if err := ensureAllPortsAvailableOnIP(bindIp); err != nil {
glog.Errorf("Port allocation failed: %v", err)
os.Exit(1)
}
// Set master.peers to "none" if not specified (single master mode)
if *miniMasterOptions.peers == "" {
*miniMasterOptions.peers = "none"
@@ -325,6 +757,20 @@ func runMini(cmd *Command, args []string) bool {
}
miniFilerOptions.defaultLevelDbDirectory = miniMasterOptions.metaFolder
// Calculate and set optimal volume size limit based on available disk space
// Only auto-calculate if user didn't explicitly specify a value via -master.volumeSizeLimitMB
if !isFlagPassed("master.volumeSizeLimitMB") {
// User didn't override, use auto-calculated value
// The -dir flag can accept comma-separated directories; use the first one for disk space calculation
resolvedDataFolder := util.ResolvePath(util.StringSplit(*miniDataFolders, ",")[0])
optimalVolumeSizeMB := calculateOptimalVolumeSizeMB(resolvedDataFolder)
miniMasterOptions.volumeSizeLimitMB = &optimalVolumeSizeMB
glog.Infof("Mini started with auto-calculated optimal volume size limit: %dMB", optimalVolumeSizeMB)
} else {
// User specified a custom value
glog.Infof("Mini started with user-specified volume size limit: %dMB", *miniMasterOptions.volumeSizeLimitMB)
}
miniWhiteList := util.StringSplit(*miniWhiteListOption, ",")
// Start all services with proper dependency coordination
@@ -338,18 +784,24 @@ func runMini(cmd *Command, args []string) bool {
// Print welcome message after all services are running
printWelcomeMessage()
// Save configuration to file for persistence and documentation
saveMiniConfiguration(*miniDataFolders)
select {}
}
// startMiniServices starts all mini services with proper dependency coordination
func startMiniServices(miniWhiteList []string, allServicesReady chan struct{}) {
// Determine bind IP for health checks
bindIp := getBindIp()
// Start Master server (no dependencies)
go startMiniService("Master", func() {
startMaster(miniMasterOptions, miniWhiteList)
}, *miniMasterOptions.port)
// Wait for master to be ready
waitForServiceReady("Master", *miniMasterOptions.port)
waitForServiceReady("Master", *miniMasterOptions.port, bindIp)
// Start Volume server (depends on master)
go startMiniService("Volume", func() {
@@ -358,7 +810,7 @@ func startMiniServices(miniWhiteList []string, allServicesReady chan struct{}) {
}, *miniOptions.v.port)
// Wait for volume to be ready
waitForServiceReady("Volume", *miniOptions.v.port)
waitForServiceReady("Volume", *miniOptions.v.port, bindIp)
// Start Filer (depends on master and volume)
go startMiniService("Filer", func() {
@@ -366,7 +818,7 @@ func startMiniServices(miniWhiteList []string, allServicesReady chan struct{}) {
}, *miniFilerOptions.port)
// Wait for filer to be ready
waitForServiceReady("Filer", *miniFilerOptions.port)
waitForServiceReady("Filer", *miniFilerOptions.port, bindIp)
// Start S3 and WebDAV in parallel (both depend on filer)
go startMiniService("S3", func() {
@@ -378,8 +830,8 @@ func startMiniServices(miniWhiteList []string, allServicesReady chan struct{}) {
}, *miniWebDavOptions.port)
// Wait for both S3 and WebDAV to be ready
waitForServiceReady("S3", *miniS3Options.port)
waitForServiceReady("WebDAV", *miniWebDavOptions.port)
waitForServiceReady("S3", *miniS3Options.port, bindIp)
waitForServiceReady("WebDAV", *miniWebDavOptions.port, bindIp)
// Start Admin with worker (depends on master, filer, S3, WebDAV)
go startMiniAdminWithWorker(allServicesReady)
@@ -392,8 +844,8 @@ func startMiniService(name string, fn func(), port int) {
}
// waitForServiceReady pings the service HTTP endpoint to check if it's ready to accept connections
func waitForServiceReady(name string, port int) {
address := fmt.Sprintf("http://127.0.0.1:%d", port)
func waitForServiceReady(name string, port int, bindIp string) {
address := fmt.Sprintf("http://%s:%d", bindIp, port)
maxAttempts := 30 // 30 * 200ms = 6 seconds max wait
attempt := 0
client := &http.Client{
@@ -427,6 +879,7 @@ func startS3Service() {
iamCfg := &iam_pb.S3ApiConfiguration{}
ident := &iam_pb.Identity{Name: user}
ident.Credentials = append(ident.Credentials, &iam_pb.Credential{AccessKey: accessKey, SecretKey: secretKey})
ident.Actions = append(ident.Actions, "Admin")
iamCfg.Identities = append(iamCfg.Identities, ident)
iamPath := filepath.Join(*miniDataFolders, "iam_config.json")
@@ -470,8 +923,28 @@ func startMiniAdminWithWorker(allServicesReady chan struct{}) {
// Set admin options
*miniAdminOptions.master = masterAddr
// Security validation: prevent empty username when password is set
if *miniAdminOptions.adminPassword != "" && *miniAdminOptions.adminUser == "" {
glog.Fatalf("Error: -admin.user cannot be empty when -admin.password is set")
}
if *miniAdminOptions.readOnlyPassword != "" && *miniAdminOptions.readOnlyUser == "" {
glog.Fatalf("Error: -admin.readOnlyUser is required when -admin.readOnlyPassword is set")
}
// Security validation: prevent username conflicts between admin and read-only users
if *miniAdminOptions.adminUser != "" && *miniAdminOptions.readOnlyUser != "" &&
*miniAdminOptions.adminUser == *miniAdminOptions.readOnlyUser {
glog.Fatalf("Error: -admin.user and -admin.readOnlyUser must be different when both are configured")
}
// Security validation: admin password is required for read-only user
if *miniAdminOptions.readOnlyPassword != "" && *miniAdminOptions.adminPassword == "" {
glog.Fatalf("Error: -admin.password must be set when -admin.readOnlyPassword is configured")
}
// gRPC port should have been initialized by ensureAllPortsAvailableOnIP in runMini
// If it's still 0, that indicates a problem with the port initialization sequence
if *miniAdminOptions.grpcPort == 0 {
*miniAdminOptions.grpcPort = *miniAdminOptions.port + 10000
glog.Fatalf("Admin gRPC port was not initialized before startAdminServer. This indicates a problem with the port initialization sequence.")
}
// Create data directory if specified
@@ -609,6 +1082,8 @@ func startMiniWorker() {
// Set admin client
workerInstance.SetAdminClient(adminClient)
// Metrics server is already started in the main init function above, so no need to start it again here
// Start the worker
err = workerInstance.Start()
if err != nil {
@@ -633,7 +1108,7 @@ const welcomeMessageTemplate = `
Volume Server: http://%s:%d
Optimized Settings:
• Volume size limit: 128MB
• Volume size limit: %dMB
• Volume max: auto (based on free disk space)
• Pre-stop seconds: 1 (faster shutdown)
• Master peers: none (single master mode)
@@ -673,6 +1148,7 @@ func printWelcomeMessage() {
*miniIp, *miniWebDavOptions.port,
*miniIp, *miniAdminOptions.port,
*miniIp, *miniOptions.v.port,
*miniMasterOptions.volumeSizeLimitMB,
*miniDataFolders,
)
weed/command/s3.go
+3
View File
@@ -62,6 +62,7 @@ type S3Options struct {
enableIam *bool
debug *bool
debugPort *int
cipher *bool
}
func init() {
@@ -93,6 +94,7 @@ func init() {
s3StandaloneOptions.enableIam = cmdS3.Flag.Bool("iam", true, "enable embedded IAM API on the same port")
s3StandaloneOptions.debug = cmdS3.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port")
s3StandaloneOptions.debugPort = cmdS3.Flag.Int("debug.port", 6060, "http port for debugging")
s3StandaloneOptions.cipher = cmdS3.Flag.Bool("encryptVolumeData", false, "encrypt data on volume servers")
}
var cmdS3 = &Command{
@@ -290,6 +292,7 @@ func (s3opt *S3Options) startS3Server() bool {
ConcurrentUploadLimit: int64(*s3opt.concurrentUploadLimitMB) * 1024 * 1024,
ConcurrentFileUploadLimit: int64(*s3opt.concurrentFileUploadLimit),
EnableIam: *s3opt.enableIam, // Embedded IAM API (enabled by default)
Cipher: *s3opt.cipher, // encrypt data on volume servers
})
if s3ApiServer_err != nil {
glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
weed/command/server.go
+1
View File
@@ -175,6 +175,7 @@ func init() {
s3Options.concurrentUploadLimitMB = cmdServer.Flag.Int("s3.concurrentUploadLimitMB", 0, "limit total concurrent upload size for S3, 0 means unlimited")
s3Options.concurrentFileUploadLimit = cmdServer.Flag.Int("s3.concurrentFileUploadLimit", 0, "limit number of concurrent file uploads for S3, 0 means unlimited")
s3Options.enableIam = cmdServer.Flag.Bool("s3.iam", true, "enable embedded IAM API on the same S3 port")
s3Options.cipher = cmdServer.Flag.Bool("s3.encryptVolumeData", false, "encrypt data on volume servers for S3 uploads")
sftpOptions.port = cmdServer.Flag.Int("sftp.port", 2022, "SFTP server listen port")
sftpOptions.sshPrivateKey = cmdServer.Flag.String("sftp.sshPrivateKey", "", "path to the SSH private key file for host authentication")

Some files were not shown because too many files have changed in this diff Show More