diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index 9e3a0a451..8dedfd26f 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -223,5 +223,23 @@ jobs: echo "✓ Successfully copied ${{ matrix.variant }} to Docker Hub" + helm-release: + runs-on: ubuntu-latest + needs: [copy-to-dockerhub] + permissions: + contents: write + pages: write + steps: + - uses: actions/checkout@v6 + - name: Publish Helm charts + uses: stefanprodan/helm-gh-pages@v1.7.0 + with: + token: ${{ secrets.GITHUB_TOKEN }} + charts_dir: k8s/charts + target_dir: helm + branch: gh-pages + helm_version: "3.18.4" + + diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 34c393cee..8dcc829a8 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -33,6 +33,15 @@ jobs: run: | cd weed; go get -v -t -d ./... + - name: Go Vet (excluding protobuf lock copying) + run: | + cd weed + # Run go vet and filter out known protobuf MessageState lock copying warnings + # These are expected in generated protobuf code with embedded sync.Mutex and are safe in practice + go vet -v ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-output.txt + # Fail only if there are actual vet errors (not counting the filtered lock warnings) + if grep -q "vet:" vet-output.txt; then exit 1; fi + - name: Build run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v . diff --git a/.github/workflows/helm_chart_release.yml b/.github/workflows/helm_chart_release.yml deleted file mode 100644 index d21b00451..000000000 --- a/.github/workflows/helm_chart_release.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: "helm: publish charts" -on: - push: - tags: - - '*' - -permissions: - contents: write - pages: write - -jobs: - release: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 - - name: Publish Helm charts - uses: stefanprodan/helm-gh-pages@v1.7.0 - with: - token: ${{ secrets.GITHUB_TOKEN }} - charts_dir: k8s/charts - target_dir: helm - branch: gh-pages - helm_version: "3.18.4" diff --git a/.github/workflows/kafka-tests.yml b/.github/workflows/kafka-tests.yml index 7da13a443..1b4bed572 100644 --- a/.github/workflows/kafka-tests.yml +++ b/.github/workflows/kafka-tests.yml @@ -52,6 +52,7 @@ jobs: run: | cd test/kafka go mod download + go mod tidy - name: Run Kafka Gateway Unit Tests run: | @@ -96,6 +97,7 @@ jobs: run: | cd test/kafka go mod download + go mod tidy - name: Run Integration Tests run: | @@ -154,6 +156,7 @@ jobs: cd test/kafka # Use go mod download with timeout to prevent hanging timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules" + go mod tidy - name: Build and start SeaweedFS MQ run: | @@ -332,6 +335,7 @@ jobs: cd test/kafka # Use go mod download with timeout to prevent hanging timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules" + go mod tidy - name: Build and start SeaweedFS MQ run: | @@ -492,6 +496,7 @@ jobs: run: | cd test/kafka timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules" + go mod tidy - name: Build and start SeaweedFS MQ run: | @@ -649,6 +654,7 @@ jobs: run: | cd test/kafka timeout 90s go mod download || echo "Warning: Dependency download timed out, continuing with cached modules" + go mod tidy - name: Build and start SeaweedFS MQ run: | @@ -803,6 +809,7 @@ jobs: run: | cd test/kafka go mod download + go mod tidy - name: Run Protocol Tests run: | diff --git a/.github/workflows/s3-sse-tests.yml b/.github/workflows/s3-sse-tests.yml index 946e4735e..2a8c0b332 100644 --- a/.github/workflows/s3-sse-tests.yml +++ b/.github/workflows/s3-sse-tests.yml @@ -345,3 +345,53 @@ jobs: name: s3-sse-performance-logs path: test/s3/sse/weed-test*.log retention-days: 7 + + s3-volume-encryption: + name: S3 Volume Encryption Test + runs-on: ubuntu-22.04 + timeout-minutes: 20 + + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + id: go + + - name: Install SeaweedFS + run: | + go install -buildvcs=false + + - name: Run S3 Volume Encryption Integration Tests + timeout-minutes: 15 + working-directory: test/s3/sse + run: | + set -x + echo "=== System Information ===" + uname -a + free -h + + # Run volume encryption tests with -s3.encryptVolumeData flag + echo "🚀 Running S3 volume encryption integration tests..." + make test-volume-encryption || { + echo "❌ Volume encryption tests failed, checking logs..." + if [ -f /tmp/seaweedfs-sse-mini.log ]; then + echo "=== Server logs ===" + tail -100 /tmp/seaweedfs-sse-mini.log + fi + echo "=== Process information ===" + ps aux | grep -E "(weed|test)" || true + exit 1 + } + + - name: Upload server logs on failure + if: failure() + uses: actions/upload-artifact@v6 + with: + name: s3-volume-encryption-logs + path: /tmp/seaweedfs-sse-*.log + retention-days: 3 + diff --git a/.github/workflows/test-s3-over-https-using-awscli.yml b/.github/workflows/test-s3-over-https-using-awscli.yml index 249d0c9e9..cf7efd7ab 100644 --- a/.github/workflows/test-s3-over-https-using-awscli.yml +++ b/.github/workflows/test-s3-over-https-using-awscli.yml @@ -18,7 +18,7 @@ defaults: jobs: awscli-tests: runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 steps: - uses: actions/checkout@v6 diff --git a/.gitignore b/.gitignore index 91fa9391d..10bc81f63 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,9 @@ coverage.out test/s3/remote_cache/remote-server.pid test/s3/remote_cache/primary-server.pid /test/erasure_coding/filerldb2 +/test/s3/cors/test-mini-data +/test/s3/filer_group/test-volume-data + +# ID and PID files +*.id +*.pid diff --git a/go.mod b/go.mod index c1407f165..30cbc98a5 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/seaweedfs/seaweedfs -go 1.24.0 - -toolchain go1.24.1 +go 1.24.9 require ( cloud.google.com/go v0.121.6 // indirect @@ -36,7 +34,7 @@ require ( github.com/golang/snappy v1.0.0 github.com/google/btree v1.1.3 github.com/google/uuid v1.6.0 - github.com/google/wire v0.6.0 // indirect + github.com/google/wire v0.7.0 // indirect github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gorilla/mux v1.8.1 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect @@ -93,27 +91,27 @@ require ( go.etcd.io/etcd/client/v3 v3.6.6 go.mongodb.org/mongo-driver v1.17.6 go.opencensus.io v0.24.0 // indirect - gocloud.dev v0.43.0 - gocloud.dev/pubsub/natspubsub v0.43.0 + gocloud.dev v0.44.0 + gocloud.dev/pubsub/natspubsub v0.44.0 gocloud.dev/pubsub/rabbitpubsub v0.43.0 - golang.org/x/crypto v0.45.0 + golang.org/x/crypto v0.46.0 golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 golang.org/x/image v0.34.0 - golang.org/x/net v0.47.0 - golang.org/x/oauth2 v0.32.0 // indirect - golang.org/x/sys v0.38.0 + golang.org/x/net v0.48.0 + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sys v0.39.0 golang.org/x/text v0.32.0 // indirect golang.org/x/tools v0.39.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect - google.golang.org/api v0.247.0 + google.golang.org/api v0.258.0 google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 // indirect google.golang.org/grpc v1.77.0 - google.golang.org/protobuf v1.36.10 + google.golang.org/protobuf v1.36.11 gopkg.in/inf.v0 v0.9.1 // indirect modernc.org/b v1.0.0 // indirect modernc.org/mathutil v1.7.1 modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.39.0 + modernc.org/sqlite v1.42.2 modernc.org/strutil v1.2.1 ) @@ -127,8 +125,8 @@ require ( github.com/arangodb/go-driver v1.6.9 github.com/armon/go-metrics v0.4.1 github.com/aws/aws-sdk-go-v2 v1.41.0 - github.com/aws/aws-sdk-go-v2/config v1.31.3 - github.com/aws/aws-sdk-go-v2/credentials v1.19.5 + github.com/aws/aws-sdk-go-v2/config v1.32.6 + github.com/aws/aws-sdk-go-v2/credentials v1.19.6 github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 github.com/cognusion/imaging v1.0.2 github.com/fluent/fluent-logger-golang v1.10.1 @@ -147,18 +145,18 @@ require ( github.com/mattn/go-sqlite3 v1.14.32 github.com/minio/crc64nvme v1.1.1 github.com/orcaman/concurrent-map/v2 v2.0.1 - github.com/parquet-go/parquet-go v0.25.1 + github.com/parquet-go/parquet-go v0.26.3 github.com/pkg/sftp v1.13.10 github.com/rabbitmq/amqp091-go v1.10.0 github.com/rclone/rclone v1.71.2 github.com/rdleal/intervalst v1.5.0 github.com/redis/go-redis/v9 v9.17.2 - github.com/schollz/progressbar/v3 v3.18.0 + github.com/schollz/progressbar/v3 v3.19.0 github.com/shirou/gopsutil/v4 v4.25.11 github.com/tarantool/go-tarantool/v2 v2.4.1 github.com/tikv/client-go/v2 v2.0.7 github.com/xeipuuv/gojsonschema v1.2.0 - github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 + github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0 go.etcd.io/etcd/client/pkg/v3 v3.6.6 go.uber.org/atomic v1.11.0 @@ -171,8 +169,9 @@ require github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // ind require ( cloud.google.com/go/longrunning v0.6.7 // indirect - cloud.google.com/go/pubsub/v2 v2.0.0 // indirect + cloud.google.com/go/pubsub/v2 v2.2.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect github.com/bazelbuild/rules_go v0.46.0 // indirect github.com/biogo/store v0.0.0-20201120204734-aad293a2328f // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect @@ -201,6 +200,8 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/openzipkin/zipkin-go v0.4.3 // indirect + github.com/parquet-go/bitpack v1.0.0 // indirect + github.com/parquet-go/jsonlite v1.0.0 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pierrre/geohash v1.0.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect @@ -226,7 +227,7 @@ require ( require ( cel.dev/expr v0.24.0 // indirect - cloud.google.com/go/auth v0.16.5 // indirect + cloud.google.com/go/auth v0.17.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect cloud.google.com/go/iam v1.5.2 // indirect @@ -261,10 +262,10 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect @@ -272,7 +273,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect github.com/aws/smithy-go v1.24.0 // indirect @@ -327,7 +328,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect github.com/gorilla/context v1.1.2 // indirect github.com/gorilla/schema v1.4.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect @@ -445,15 +446,15 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/arch v0.20.0 // indirect - golang.org/x/term v0.37.0 // indirect - golang.org/x/time v0.12.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/time v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/validator.v2 v2.0.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - modernc.org/libc v1.66.3 // indirect + modernc.org/libc v1.66.10 // indirect moul.io/http2curl/v2 v2.3.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect storj.io/common v0.0.0-20250808122759-804533d519c1 // indirect diff --git a/go.sum b/go.sum index 818406722..63b407cfa 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVo cloud.google.com/go/assuredworkloads v1.8.0/go.mod h1:AsX2cqyNCOvEQC8RMPnoc0yEarXQk6WEKkxYfL6kGIo= cloud.google.com/go/assuredworkloads v1.9.0/go.mod h1:kFuI1P78bplYtT77Tb1hi0FMxM0vVpRC7VVoJC3ZoT0= cloud.google.com/go/assuredworkloads v1.10.0/go.mod h1:kwdUQuXcedVdsIaKgKTp9t0UJkE5+PAVNhdQm4ZVq2E= -cloud.google.com/go/auth v0.16.5 h1:mFWNQ2FEVWAliEQWpAdH80omXFokmrnbDhUS9cBywsI= -cloud.google.com/go/auth v0.16.5/go.mod h1:utzRfHMP+Vv0mpOkTRQoWD2q3BatTOoWbA7gCc2dUhQ= +cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4= +cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ= cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= @@ -385,8 +385,8 @@ cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhz cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM= cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk= -cloud.google.com/go/pubsub/v2 v2.0.0 h1:0qS6mRJ41gD1lNmM/vdm6bR7DQu6coQcVwD+VPf0Bz0= -cloud.google.com/go/pubsub/v2 v2.0.0/go.mod h1:0aztFxNzVQIRSZ8vUr79uH2bS3jwLebwK6q1sgEub+E= +cloud.google.com/go/pubsub/v2 v2.2.1 h1:3brZcshL3fIiD1qOxAE2QW9wxsfjioy014x4yC9XuYI= +cloud.google.com/go/pubsub/v2 v2.2.1/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw= cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg= cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k= cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= @@ -669,20 +669,20 @@ github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgP github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= -github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco= -github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE= -github.com/aws/aws-sdk-go-v2/credentials v1.19.5 h1:xMo63RlqP3ZZydpJDMBsH9uJ10hgHYfQFIk1cHDXrR4= -github.com/aws/aws-sdk-go-v2/credentials v1.19.5/go.mod h1:hhbH6oRcou+LpXfA/0vPElh/e0M3aFeOblE1sssAAEk= +github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= +github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3 h1:4GNV1lhyELGjMz5ILMRxDvxvOaeo3Ux9Z69S1EgVMMQ= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.3/go.mod h1:br7KA6edAAqDGUYJ+zVVPAyMrPhnN+zdt17yTUT6FPw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= @@ -695,12 +695,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1c github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0= github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI= github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE= github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U= github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ= github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.7 h1:eYnlt6QxnFINKzwxP5/Ucs1vkG7VT3Iezmvfgc2waUw= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.7/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0= github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70= @@ -1132,22 +1134,21 @@ github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/wire v0.6.0 h1:HBkoIh4BdSxoyo9PveV8giw7ZsaBOvzWKfcg/6MrVwI= -github.com/google/wire v0.6.0/go.mod h1:F4QhpQ9EDIdJ1Mbop/NZBRB+5yrR6qg3BnctaoUk6NA= +github.com/google/wire v0.7.0 h1:JxUKI6+CVBgCO2WToKy/nQk0sS+amI9z9EjVmdaocj4= +github.com/google/wire v0.7.0/go.mod h1:n6YbUQD9cPKTnHXEBN2DXlOp/mVADhVErcMFb0v3J18= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= -github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= -github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAVrGgAa0f2/R35S4DJwfFaUPFQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -1469,8 +1470,12 @@ github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q github.com/ory/dockertest/v3 v3.6.0/go.mod h1:4ZOpj8qBUmh8fcBSVzkH2bws2s91JdGvHUqan4GHEuQ= github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg= github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= -github.com/parquet-go/parquet-go v0.25.1 h1:l7jJwNM0xrk0cnIIptWMtnSnuxRkwq53S+Po3KG8Xgo= -github.com/parquet-go/parquet-go v0.25.1/go.mod h1:AXBuotO1XiBtcqJb/FKFyjBG4aqa3aQAAWF3ZPzCanY= +github.com/parquet-go/bitpack v1.0.0 h1:AUqzlKzPPXf2bCdjfj4sTeacrUwsT7NlcYDMUQxPcQA= +github.com/parquet-go/bitpack v1.0.0/go.mod h1:XnVk9TH+O40eOOmvpAVZ7K2ocQFrQwysLMnc6M/8lgs= +github.com/parquet-go/jsonlite v1.0.0 h1:87QNdi56wOfsE5bdgas0vRzHPxfJgzrXGml1zZdd7VU= +github.com/parquet-go/jsonlite v1.0.0/go.mod h1:nDjpkpL4EOtqs6NQugUsi0Rleq9sW/OtC1NnZEnxzF0= +github.com/parquet-go/parquet-go v0.26.3 h1:kJY+xmjcR7BH77tyHqasJpIl3kch/6EIO3TW4tFj69M= +github.com/parquet-go/parquet-go v0.26.3/go.mod h1:h9GcSt41Knf5qXI1tp1TfR8bDBUtvdUMzSKe26aZcHk= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -1612,8 +1617,8 @@ github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI= github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= -github.com/schollz/progressbar/v3 v3.18.0 h1:uXdoHABRFmNIjUfte/Ex7WtuyVslrw2wVPQmCN62HpA= -github.com/schollz/progressbar/v3 v3.18.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= +github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= +github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138 h1:bX1vBF7GQjPeFQsCAZ8gCQGS/nJQnekL7gZ4Qg/pF4E= github.com/seaweedfs/cockroachdb-parser v0.0.0-20251021184156-909763b17138/go.mod h1:JSKCh6uCHBz91lQYFYHCyTrSVIPge4SUFVn28iwMNB0= github.com/seaweedfs/goexif v1.0.3 h1:ve/OjI7dxPW8X9YQsv3JuVMaxEyF9Rvfd04ouL+Bz30= @@ -1777,8 +1782,8 @@ github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mo github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb h1:LZ6dhVfWzhicf/P5Xh7fA0Jd7rfGduxmB2QZpD+Lz9Q= github.com/ydb-platform/ydb-go-genproto v0.0.0-20251125145508-6d7ef87db5cb/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= -github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0 h1:/NyPd9KnCJgzrEXCArqk1ThqCH2Dh31uUwl88o/VkuM= -github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.0/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM= +github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1 h1:XaRxeVrOyl3y6v9CiYMWaFdZ6zevvYe+TRxOR8ifa2s= +github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.1/go.mod h1:9YzkhlIymWaJGX6KMU3vh5sOf3UKbCXkG/ZdjaI3zNM= github.com/ydb-platform/ydb-go-sdk/v3 v3.44.0/go.mod h1:oSLwnuilwIpaF5bJJMAofnGgzPJusoI3zWMNb8I+GnM= github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0= github.com/ydb-platform/ydb-go-sdk/v3 v3.122.0 h1:uvqheUfoEJz0CLLLaYct69wpvoaosM/2joWFcbwoqfw= @@ -1886,10 +1891,10 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -gocloud.dev v0.43.0 h1:aW3eq4RMyehbJ54PMsh4hsp7iX8cO/98ZRzJJOzN/5M= -gocloud.dev v0.43.0/go.mod h1:eD8rkg7LhKUHrzkEdLTZ+Ty/vgPHPCd+yMQdfelQVu4= -gocloud.dev/pubsub/natspubsub v0.43.0 h1:k35tFoaorvD9Fa26zVEEzyXiMOEyXNHc0pBOmRYvQI0= -gocloud.dev/pubsub/natspubsub v0.43.0/go.mod h1:xJn8TO8pGYieDn6AsRFsYfhQW8cnC+xGmG9APGNxkpQ= +gocloud.dev v0.44.0 h1:iVyMAqFl2r6xUy7M4mfqwlN+21UpJoEtgHEcfiLMUXs= +gocloud.dev v0.44.0/go.mod h1:ZmjROXGdC/eKZLF1N+RujDlFRx3D+4Av2thREKDMVxY= +gocloud.dev/pubsub/natspubsub v0.44.0 h1:1Us76ckkdgtiE1p1rJZ+38b9TQP051bmjAiQlFQzYrM= +gocloud.dev/pubsub/natspubsub v0.44.0/go.mod h1:PvVAGIhL14PWGwWIXX/zAK42ixr2/PKP4Q4yMiAUraQ= gocloud.dev/pubsub/rabbitpubsub v0.43.0 h1:6nNZFSlJ1dk2GujL8PFltfLz3vC6IbrpjGS4FTduo1s= gocloud.dev/pubsub/rabbitpubsub v0.43.0/go.mod h1:sEaueAGat+OASRoB3QDkghCtibKttgg7X6zsPTm1pl0= golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= @@ -1911,14 +1916,13 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1982,7 +1986,6 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= @@ -2052,12 +2055,11 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2086,8 +2088,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= -golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2212,14 +2214,13 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2231,14 +2232,13 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2267,8 +2267,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2339,7 +2339,6 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= @@ -2420,8 +2419,8 @@ google.golang.org/api v0.106.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/ google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= -google.golang.org/api v0.247.0 h1:tSd/e0QrUlLsrwMKmkbQhYVa109qIintOls2Wh6bngc= -google.golang.org/api v0.247.0/go.mod h1:r1qZOPmxXffXg6xS5uhx16Fa/UFY8QU/K4bfKrnvovM= +google.golang.org/api v0.258.0 h1:IKo1j5FBlN74fe5isA2PVozN3Y5pwNKriEgAXPOkDAc= +google.golang.org/api v0.258.0/go.mod h1:qhOMTQEZ6lUps63ZNq9jhODswwjkjYYguA7fA3TBFww= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2559,8 +2558,8 @@ google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79 h1:Nt6z9UHqSlIdIGJ google.golang.org/genproto v0.0.0-20250715232539-7130f93afb79/go.mod h1:kTmlBHMPqR5uCZPBvwa2B18mvubkjyY3CRLI0c6fj0s= google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4= google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2624,8 +2623,8 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2676,19 +2675,19 @@ modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/cc/v3 v3.36.2/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= -modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM= -modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= +modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= modernc.org/ccgo/v3 v3.0.0-20220428102840-41399a37e894/go.mod h1:eI31LL8EwEBKPpNpA4bU1/i+sKOwOrQy8D87zWUcRZc= modernc.org/ccgo/v3 v3.0.0-20220430103911-bc99d88307be/go.mod h1:bwdAnOoaIt8Ax9YdWGjxWsdkPcZyRPHqrOvJxaKAKGw= modernc.org/ccgo/v3 v3.16.4/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= modernc.org/ccgo/v3 v3.16.6/go.mod h1:tGtX0gE9Jn7hdZFeU88slbTh1UtCYKusWOoCJuvkWsQ= modernc.org/ccgo/v3 v3.16.8/go.mod h1:zNjwkizS+fIFDrDjIAgBSCLkWbJuHF+ar3QRn+Z9aws= modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= -modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= -modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= +modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= +modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= -modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM= -modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= @@ -2701,8 +2700,8 @@ modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= -modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= -modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= +modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= @@ -2721,8 +2720,8 @@ modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= -modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY= -modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +modernc.org/sqlite v1.42.2 h1:7hkZUNJvJFN2PgfUdjni9Kbvd4ef4mNLOu0B9FGxM74= +modernc.org/sqlite v1.42.2/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8= modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= diff --git a/k8s/charts/seaweedfs/Chart.yaml b/k8s/charts/seaweedfs/Chart.yaml index 234187cce..6dae918e1 100644 --- a/k8s/charts/seaweedfs/Chart.yaml +++ b/k8s/charts/seaweedfs/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 description: SeaweedFS name: seaweedfs -appVersion: "4.03" +appVersion: "4.04" # Dev note: Trigger a helm chart release by `git tag -a helm-` -version: 4.0.403 +version: 4.0.404 diff --git a/k8s/charts/seaweedfs/templates/volume/volume-statefulset.yaml b/k8s/charts/seaweedfs/templates/volume/volume-statefulset.yaml index 6a551a6c9..045b95c2e 100644 --- a/k8s/charts/seaweedfs/templates/volume/volume-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/volume/volume-statefulset.yaml @@ -176,6 +176,9 @@ spec: {{- if $volume.dataCenter }} -dataCenter={{ $volume.dataCenter }} \ {{- end }} + {{- if $volume.id }} + -id={{ $volume.id }} \ + {{- end }} -ip.bind={{ $volume.ipBind }} \ -readMode={{ $volume.readMode }} \ {{- if $volume.whiteList }} diff --git a/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml b/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml index 60f608702..d57a900d5 100644 --- a/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/worker/worker-deployment.yaml @@ -134,11 +134,14 @@ spec: {{- if .Values.worker.adminServer }} -admin={{ .Values.worker.adminServer }} \ {{- else }} - -admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.grpcPort }} \ + -admin={{ template "seaweedfs.name" . }}-admin.{{ .Release.Namespace }}:{{ .Values.admin.port }} \ {{- end }} -capabilities={{ .Values.worker.capabilities }} \ -maxConcurrent={{ .Values.worker.maxConcurrent }} \ - -workingDir={{ .Values.worker.workingDir }}{{- if .Values.worker.extraArgs }} \{{ end }} + -workingDir={{ .Values.worker.workingDir }}{{- if or .Values.worker.metricsPort .Values.worker.extraArgs }} \{{ end }} + {{- if .Values.worker.metricsPort }} + -metricsPort={{ .Values.worker.metricsPort }}{{- if .Values.worker.extraArgs }} \{{ end }} + {{- end }} {{- range $index, $arg := .Values.worker.extraArgs }} {{ $arg }}{{- if lt $index (sub (len $.Values.worker.extraArgs) 1) }} \{{ end }} {{- end }} @@ -187,9 +190,13 @@ spec: {{- end }} {{- if .Values.worker.livenessProbe.enabled }} livenessProbe: - {{- with .Values.worker.livenessProbe.tcpSocket }} + {{- if .Values.worker.livenessProbe.httpGet }} + httpGet: + path: {{ .Values.worker.livenessProbe.httpGet.path }} + port: {{ .Values.worker.livenessProbe.httpGet.port }} + {{- else if .Values.worker.livenessProbe.tcpSocket }} tcpSocket: - port: {{ .port }} + port: {{ .Values.worker.livenessProbe.tcpSocket.port }} {{- end }} initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }} @@ -199,9 +206,13 @@ spec: {{- end }} {{- if .Values.worker.readinessProbe.enabled }} readinessProbe: - {{- with .Values.worker.readinessProbe.tcpSocket }} + {{- if .Values.worker.readinessProbe.httpGet }} + httpGet: + path: {{ .Values.worker.readinessProbe.httpGet.path }} + port: {{ .Values.worker.readinessProbe.httpGet.port }} + {{- else if .Values.worker.readinessProbe.tcpSocket }} tcpSocket: - port: {{ .port }} + port: {{ .Values.worker.readinessProbe.tcpSocket.port }} {{- end }} initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index b03e66c40..a94d7f183 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -401,6 +401,10 @@ volume: # Volume server's rack name rack: null + # Stable identifier for the volume server, independent of IP address + # Useful for Kubernetes environments with hostPath volumes to maintain stable identity + id: null + # Volume server's data center name dataCenter: null @@ -1302,10 +1306,11 @@ worker: extraEnvironmentVars: {} # Health checks for worker pods - # Since workers do not have an HTTP endpoint, a tcpSocket probe on the metrics port is recommended. + # Workers expose /health (liveness) and /ready (readiness) endpoints on the metricsPort livenessProbe: enabled: true - tcpSocket: + httpGet: + path: /health port: metrics initialDelaySeconds: 30 periodSeconds: 60 @@ -1315,7 +1320,8 @@ worker: readinessProbe: enabled: true - tcpSocket: + httpGet: + path: /ready port: metrics initialDelaySeconds: 20 periodSeconds: 15 diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index f8334b734..e559431dd 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -87,7 +87,7 @@ public class FilerGrpcClient { .maxHeaderListSize(16 * 1024 * 1024) .keepAliveTime(KEEP_ALIVE_TIME_SECONDS, TimeUnit.SECONDS) .keepAliveTimeout(KEEP_ALIVE_TIMEOUT_SECONDS, TimeUnit.SECONDS) - .keepAliveWithoutCalls(true) + .keepAliveWithoutCalls(false) .withOption(io.grpc.netty.shaded.io.netty.channel.ChannelOption.SO_RCVBUF, 16 * 1024 * 1024) .withOption(io.grpc.netty.shaded.io.netty.channel.ChannelOption.SO_SNDBUF, 16 * 1024 * 1024); diff --git a/test/fuse_integration/Makefile b/test/fuse_integration/Makefile index fe2ad690b..3c1e68d59 100644 --- a/test/fuse_integration/Makefile +++ b/test/fuse_integration/Makefile @@ -12,20 +12,20 @@ COVERAGE_FILE := coverage.out # Check if weed binary exists check-binary: @if [ ! -f "$(WEED_BINARY)" ]; then \ - echo "❌ SeaweedFS binary not found at $(WEED_BINARY)"; \ + echo "ERROR SeaweedFS binary not found at $(WEED_BINARY)"; \ echo " Please run 'make' in the root directory first"; \ exit 1; \ fi - @echo "✅ SeaweedFS binary found" + @echo "OK SeaweedFS binary found" # Check FUSE installation check-fuse: @if command -v fusermount >/dev/null 2>&1; then \ - echo "✅ FUSE is installed (Linux)"; \ + echo "OK FUSE is installed (Linux)"; \ elif command -v umount >/dev/null 2>&1 && [ "$$(uname)" = "Darwin" ]; then \ - echo "✅ FUSE is available (macOS)"; \ + echo "OK FUSE is available (macOS)"; \ else \ - echo "❌ FUSE not found. Please install:"; \ + echo "ERROR FUSE not found. Please install:"; \ echo " Ubuntu/Debian: sudo apt-get install fuse"; \ echo " CentOS/RHEL: sudo yum install fuse"; \ echo " macOS: brew install macfuse"; \ @@ -36,8 +36,8 @@ check-fuse: check-go: @go version | grep -q "go1\.[2-9][0-9]" || \ go version | grep -q "go1\.2[1-9]" || \ - (echo "❌ Go $(GO_VERSION)+ required. Current: $$(go version)" && exit 1) - @echo "✅ Go version check passed" + (echo "ERROR Go $(GO_VERSION)+ required. Current: $$(go version)" && exit 1) + @echo "OK Go version check passed" # Verify all prerequisites check-prereqs: check-go check-fuse @@ -45,9 +45,9 @@ check-prereqs: check-go check-fuse # Build the SeaweedFS binary (if needed) build: - @echo "🔨 Building SeaweedFS..." + @echo "Building SeaweedFS..." cd ../.. && make - @echo "✅ Build complete" + @echo "OK Build complete" # Initialize go module (if needed) init-module: diff --git a/test/kafka/go.mod b/test/kafka/go.mod index b8f88dedd..d6c23b565 100644 --- a/test/kafka/go.mod +++ b/test/kafka/go.mod @@ -43,24 +43,25 @@ require ( github.com/appscode/go-querystring v0.0.0-20170504095604-0126cfb3f1dc // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go v1.55.8 // indirect - github.com/aws/aws-sdk-go-v2 v1.40.1 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect - github.com/aws/aws-sdk-go-v2/config v1.31.3 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.19.3 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.6 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect github.com/aws/smithy-go v1.24.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bradenaw/juniper v0.15.3 // indirect @@ -82,6 +83,7 @@ require ( github.com/cronokirby/saferith v0.33.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect @@ -229,14 +231,14 @@ require ( go.opentelemetry.io/otel/trace v1.38.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.45.0 // indirect + golang.org/x/crypto v0.46.0 // indirect golang.org/x/exp v0.0.0-20250811191247-51f88131bc50 // indirect golang.org/x/image v0.34.0 // indirect - golang.org/x/net v0.47.0 // indirect + golang.org/x/net v0.48.0 // indirect golang.org/x/oauth2 v0.32.0 // indirect golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/term v0.37.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.12.0 // indirect google.golang.org/api v0.247.0 // indirect diff --git a/test/kafka/go.sum b/test/kafka/go.sum index ea71454a8..e9e2dd40b 100644 --- a/test/kafka/go.sum +++ b/test/kafka/go.sum @@ -102,42 +102,44 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc= -github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= +github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= -github.com/aws/aws-sdk-go-v2/config v1.31.3 h1:RIb3yr/+PZ18YYNe6MDiG/3jVoJrPmdoCARwNkMGvco= -github.com/aws/aws-sdk-go-v2/config v1.31.3/go.mod h1:jjgx1n7x0FAKl6TnakqrpkHWWKcX3xfWtdnIJs5K9CE= -github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas= -github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo= +github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= +github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0= github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI= github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -202,6 +204,8 @@ github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXI github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M= github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI= github.com/dsnet/try v0.0.3/go.mod h1:WBM8tRpUmnXXhY1U6/S8dt6UWdHTQ7y8A5YSkRCkq40= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= @@ -730,8 +734,8 @@ golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -811,8 +815,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -890,8 +894,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -904,8 +908,8 @@ golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/test/kafka/loadtest/mock_million_record_test.go b/test/kafka/loadtest/mock_million_record_test.go index ada018cbb..83518fb74 100644 --- a/test/kafka/loadtest/mock_million_record_test.go +++ b/test/kafka/loadtest/mock_million_record_test.go @@ -14,6 +14,8 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/keepalive" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" "github.com/seaweedfs/seaweedfs/weed/pb/schema_pb" @@ -119,22 +121,28 @@ type PublisherSession struct { } func NewDirectBrokerClient(brokerAddr string) (*DirectBrokerClient, error) { - ctx, cancel := context.WithCancel(context.Background()) + // Use a short-lived context for dialing so we don't store a canceled + // context in the returned client. The client's operational context + // (used by methods) should be cancellable independently. + dialCtx, dialCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer dialCancel() - // Add connection timeout and keepalive settings - conn, err := grpc.DialContext(ctx, brokerAddr, + // Add keepalive settings; use exported server constants to keep values in sync. + conn, err := grpc.DialContext(dialCtx, brokerAddr, grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithTimeout(30*time.Second), grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: 30 * time.Second, // Increased from 10s to 30s - Timeout: 10 * time.Second, // Increased from 5s to 10s - PermitWithoutStream: false, // Changed to false to reduce pings + Time: pb.GrpcKeepAliveTime, // align with server MinTime + Timeout: pb.GrpcKeepAliveTimeout, // align with server timeout + PermitWithoutStream: false, // reduce pings when idle })) if err != nil { - cancel() return nil, fmt.Errorf("failed to connect to broker: %v", err) } + // Create a long-lived context for the client's lifetime and store it + // in the returned DirectBrokerClient so callers can cancel when done. + clientCtx, clientCancel := context.WithCancel(context.Background()) + client := mq_pb.NewSeaweedMessagingClient(conn) return &DirectBrokerClient{ @@ -142,8 +150,8 @@ func NewDirectBrokerClient(brokerAddr string) (*DirectBrokerClient, error) { conn: conn, client: client, publishers: make(map[string]*PublisherSession), - ctx: ctx, - cancel: cancel, + ctx: clientCtx, + cancel: clientCancel, }, nil } diff --git a/test/s3/acl/s3_acl_versioning_test.go b/test/s3/acl/s3_acl_versioning_test.go new file mode 100644 index 000000000..7e7326ec4 --- /dev/null +++ b/test/s3/acl/s3_acl_versioning_test.go @@ -0,0 +1,335 @@ +package acl + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func getS3Client(t *testing.T) *s3.Client { + endpoint := os.Getenv("S3_ENDPOINT") + if endpoint == "" { + endpoint = "http://localhost:8333" + } + accessKey := os.Getenv("S3_ACCESS_KEY") + if accessKey == "" { + accessKey = "some_access_key1" + } + secretKey := os.Getenv("S3_SECRET_KEY") + if secretKey == "" { + secretKey = "some_secret_key1" + } + + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion("us-east-1"), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + accessKey, + secretKey, + "", + )), + ) + require.NoError(t, err) + return s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = true + o.BaseEndpoint = aws.String(endpoint) + }) +} + +func createVersionedTestBucket(t *testing.T, client *s3.Client) string { + bucketName := "test-acl-versioned-" + strings.ToLower(strings.ReplaceAll(time.Now().Format("2006-01-02-15-04-05.000"), ":", "-")) + _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + + _, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucketName), + VersioningConfiguration: &types.VersioningConfiguration{ + Status: types.BucketVersioningStatusEnabled, + }, + }) + require.NoError(t, err) + time.Sleep(100 * time.Millisecond) + return bucketName +} + +func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) { + listResp, err := client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ + Bucket: aws.String(bucketName), + }) + if err == nil { + for _, obj := range listResp.Contents { + client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: obj.Key, + }) + } + } + + listVersionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucketName), + }) + if err == nil { + for _, version := range listVersionsResp.Versions { + client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: version.Key, + VersionId: version.VersionId, + }) + } + for _, marker := range listVersionsResp.DeleteMarkers { + client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: marker.Key, + VersionId: marker.VersionId, + }) + } + } + + client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) +} + +// TestGetObjectAclOnVersionedBucket tests retrieving ACL from versioned objects +func TestGetObjectAclOnVersionedBucket(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + objectKey := "versioned-object-acl" + putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Body: strings.NewReader("Hello, ACL World!"), + }) + require.NoError(t, err) + + aclResp, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + }) + require.NoError(t, err) + assert.NotNil(t, aclResp.Owner) + + t.Logf("Successfully retrieved ACL for versioned object %s (versionId: %s)", objectKey, *putResp.VersionId) +} + +// TestGetObjectAclOnSpecificVersionInVersionedBucket tests retrieving ACL for specific versions +func TestGetObjectAclOnSpecificVersionInVersionedBucket(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + objectKey := "multi-version-object-acl" + + version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Body: strings.NewReader("Version 1"), + }) + require.NoError(t, err) + + time.Sleep(50 * time.Millisecond) + + version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Body: strings.NewReader("Version 2"), + }) + require.NoError(t, err) + + versionId1 := *version1Resp.VersionId + versionId2 := *version2Resp.VersionId + + aclResp1, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + }) + require.NoError(t, err) + assert.NotNil(t, aclResp1.Owner) + + aclResp2, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + }) + require.NoError(t, err) + assert.NotNil(t, aclResp2.Owner) + + t.Logf("Successfully retrieved ACL for both versions: v1=%s, v2=%s", versionId1, versionId2) +} + +// TestPutObjectAclOnVersionedBucket tests setting ACL on versioned objects +func TestPutObjectAclOnVersionedBucket(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + objectKey := "versioned-object-put-acl" + putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Body: strings.NewReader("Hello, Put ACL!"), + }) + require.NoError(t, err) + + _, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + ACL: types.ObjectCannedACLPublicRead, + }) + require.NoError(t, err) + + aclResp, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + }) + require.NoError(t, err) + assert.NotNil(t, aclResp.Owner) + + t.Logf("Successfully set and verified ACL for versioned object %s (versionId: %s)", objectKey, *putResp.VersionId) +} + +// TestPutObjectAclOnSpecificVersionInVersionedBucket tests setting ACL on specific versions +func TestPutObjectAclOnSpecificVersionInVersionedBucket(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + objectKey := "multi-version-object-put-acl" + + version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Body: strings.NewReader("Version 1"), + }) + require.NoError(t, err) + + time.Sleep(50 * time.Millisecond) + + version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Body: strings.NewReader("Version 2"), + }) + require.NoError(t, err) + + versionId1 := *version1Resp.VersionId + versionId2 := *version2Resp.VersionId + + _, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + ACL: types.ObjectCannedACLPublicRead, + }) + require.NoError(t, err) + + _, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + ACL: types.ObjectCannedACLPrivate, + }) + require.NoError(t, err) + + aclResp1, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + }) + require.NoError(t, err) + assert.NotNil(t, aclResp1.Owner) + + aclResp2, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + }) + require.NoError(t, err) + assert.NotNil(t, aclResp2.Owner) + + t.Logf("Successfully set ACL on both versions: v1=%s, v2=%s", versionId1, versionId2) +} + +// TestModifyAclOnDifferentVersionsIndependently tests that ACL changes on one version don't affect others +func TestModifyAclOnDifferentVersionsIndependently(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + objectKey := "versioned-independent-acl" + + version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Body: strings.NewReader("Version 1"), + }) + require.NoError(t, err) + + time.Sleep(50 * time.Millisecond) + + version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Body: strings.NewReader("Version 2"), + }) + require.NoError(t, err) + + versionId1 := *version1Resp.VersionId + versionId2 := *version2Resp.VersionId + + _, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + ACL: types.ObjectCannedACLPublicRead, + }) + require.NoError(t, err) + + _, err = client.PutObjectAcl(context.TODO(), &s3.PutObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + ACL: types.ObjectCannedACLPrivate, + }) + require.NoError(t, err) + + aclRespLatest, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + }) + require.NoError(t, err) + assert.NotNil(t, aclRespLatest.Owner) + + aclResp1, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + }) + require.NoError(t, err) + assert.NotNil(t, aclResp1.Owner) + + aclResp2, err := client.GetObjectAcl(context.TODO(), &s3.GetObjectAclInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + }) + require.NoError(t, err) + assert.NotNil(t, aclResp2.Owner) + + t.Logf("Successfully verified independent ACL management across versions") +} diff --git a/test/s3/compatibility/run.sh b/test/s3/compatibility/run.sh index adfee1366..844435d69 100755 --- a/test/s3/compatibility/run.sh +++ b/test/s3/compatibility/run.sh @@ -22,10 +22,7 @@ docker stop $CONTAINER_NAME || echo "already stopped" ulimit -n 10000 # Start weed w/ filer + s3 in the background -$WEED_BIN server \ - -filer \ - -s3 \ - -volume.max 0 \ +$WEED_BIN mini \ -master.volumeSizeLimitMB 5 \ -dir "$(pwd)/tmp" \ 1>&2>weed.log & diff --git a/test/s3/copying/Makefile b/test/s3/copying/Makefile index 225798900..1a5c98c01 100644 --- a/test/s3/copying/Makefile +++ b/test/s3/copying/Makefile @@ -63,30 +63,24 @@ start-seaweedfs: check-binary @pkill -f "weed volume" || true @pkill -f "weed filer" || true @pkill -f "weed s3" || true + @pkill -f "weed mini" || true @sleep 2 # Create necessary directories - @mkdir -p /tmp/seaweedfs-test-copying-master - @mkdir -p /tmp/seaweedfs-test-copying-volume + @mkdir -p /tmp/seaweedfs-test-copying - # Start master server with volume size limit - @nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -mdir=/tmp/seaweedfs-test-copying-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-master.log 2>&1 & - @sleep 3 - - # Start volume server - @nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-copying-volume -ip=127.0.0.1 > /tmp/seaweedfs-volume.log 2>&1 & - @sleep 3 - - # Start filer server (using standard SeaweedFS gRPC port convention: HTTP port + 10000) - @nohup $(SEAWEEDFS_BINARY) filer -port=$(FILER_PORT) -port.grpc=$$(( $(FILER_PORT) + 10000 )) -master=127.0.0.1:$(MASTER_PORT) -ip=127.0.0.1 > /tmp/seaweedfs-filer.log 2>&1 & - @sleep 3 - - # Create S3 configuration - @echo '{"identities":[{"name":"$(ACCESS_KEY)","credentials":[{"accessKey":"$(ACCESS_KEY)","secretKey":"$(SECRET_KEY)"}],"actions":["Admin","Read","Write"]}]}' > /tmp/seaweedfs-s3.json - - # Start S3 server - @nohup $(SEAWEEDFS_BINARY) s3 -port=$(S3_PORT) -filer=127.0.0.1:$(FILER_PORT) -config=/tmp/seaweedfs-s3.json -ip.bind=127.0.0.1 > /tmp/seaweedfs-s3.log 2>&1 & - @sleep 5 + # Start weed mini + @echo "Starting weed mini with dir=/tmp/seaweedfs-test-copying" + @export AWS_ACCESS_KEY_ID=$(ACCESS_KEY) && \ + export AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) && \ + # Start weed mini with S3 configuration + @echo "Starting weed mini..." + @nohup $(SEAWEEDFS_BINARY) mini \ + -dir=/tmp/seaweedfs-test-copying \ + -s3.port=$(S3_PORT) \ + -s3.config=/tmp/seaweedfs-s3.json \ + -ip=127.0.0.1 \ + > /tmp/seaweedfs-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid 5 # Wait for S3 service to be ready @echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)" @@ -98,16 +92,12 @@ start-seaweedfs: check-binary echo "Waiting for S3 service... ($$i/30)"; \ sleep 1; \ done - - # Additional wait for filer gRPC to be ready - @echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)" - @sleep 2 - @echo "$(GREEN)SeaweedFS server started successfully$(NC)" - @echo "Master: http://localhost:$(MASTER_PORT)" - @echo "Volume: http://localhost:$(VOLUME_PORT)" - @echo "Filer: http://localhost:$(FILER_PORT)" + # Additional wait for filer gRPC to be ready + @echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)" + @sleep 2 + @echo "$(GREEN)SeaweedFS server started successfully$(NC)" + @echo "Mini Log: /tmp/seaweedfs-mini.log" @echo "S3: http://localhost:$(S3_PORT)" - @echo "Volume Max Size: $(VOLUME_MAX_SIZE_MB)MB" stop-seaweedfs: @echo "$(YELLOW)Stopping SeaweedFS server...$(NC)" diff --git a/test/s3/cors/Makefile b/test/s3/cors/Makefile index 3164d1341..4a1db781e 100644 --- a/test/s3/cors/Makefile +++ b/test/s3/cors/Makefile @@ -11,6 +11,7 @@ VOLUME_PORT := 8080 FILER_PORT := 8888 TEST_TIMEOUT := 10m TEST_PATTERN := TestCORS +SERVER_DIR := test-mini-data # Default target help: @@ -41,21 +42,21 @@ build-weed: @echo "Building SeaweedFS binary..." @cd ../../../weed && go build -o weed_binary . @chmod +x $(WEED_BINARY) - @echo "✅ SeaweedFS binary built at $(WEED_BINARY)" + @echo "OK SeaweedFS binary built at $(WEED_BINARY)" check-deps: build-weed @echo "Checking dependencies..." - @echo "🔍 DEBUG: Checking Go installation..." + @echo "DEBUG: Checking Go installation..." @command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1) - @echo "🔍 DEBUG: Go version: $$(go version)" - @echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)..." + @echo "DEBUG: Go version: $$(go version)" + @echo "DEBUG: Checking binary at $(WEED_BINARY)..." @test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1) @echo "🔍 DEBUG: Binary size: $$(ls -lh $(WEED_BINARY) | awk '{print $$5}')" @echo "🔍 DEBUG: Binary permissions: $$(ls -la $(WEED_BINARY) | awk '{print $$1}')" @echo "🔍 DEBUG: Checking Go module dependencies..." @go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1) @go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1) - @echo "✅ All dependencies are available" + @echo "OK All dependencies are available" # Start SeaweedFS server for testing start-server: check-deps @@ -77,23 +78,26 @@ start-server: check-deps @echo "🔍 DEBUG: Checking config file at ../../../docker/compose/s3.json" @ls -la ../../../docker/compose/s3.json || echo "⚠️ Config file not found, continuing without it" @echo "🔍 DEBUG: Creating volume directory..." - @mkdir -p ./test-volume-data - @echo "🔍 DEBUG: Launching SeaweedFS server in background..." - @echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324" - @$(WEED_BINARY) server \ - -debug \ - -s3 \ + @mkdir -p $(SERVER_DIR) + @echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..." + @echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)" + @$(WEED_BINARY) mini \ + -dir=$(SERVER_DIR) \ -s3.port=$(S3_PORT) \ - -s3.allowDeleteBucketNotEmpty=true \ - -s3.config=../../../docker/compose/s3.json \ - -filer \ - -filer.maxMB=64 \ - -master.volumeSizeLimitMB=50 \ - -volume.max=100 \ - -dir=./test-volume-data \ - -volume.preStopSeconds=1 \ - -metricsPort=9324 \ - > weed-test.log 2>&1 & echo $$! > weed-server.pid + -s3.config=$(S3_CONFIG) \ + > weed-test.log 2>&1 & \ + echo $$! > weed-test.pid + + @echo "Waiting for S3 server to be ready..." + @for i in $$(seq 1 30); do \ + if echo | nc -z localhost $(S3_PORT); then \ + echo "S3 server is ready!"; \ + exit 0; \ + fi; \ + sleep 1; \ + done; \ + echo "S3 server failed to start"; \ + exit 1 > weed-server.pid @echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')" @echo "🔍 DEBUG: Checking if PID is still running..." @sleep 2 diff --git a/test/s3/delete/s3_multi_delete_versioning_test.go b/test/s3/delete/s3_multi_delete_versioning_test.go new file mode 100644 index 000000000..22179c611 --- /dev/null +++ b/test/s3/delete/s3_multi_delete_versioning_test.go @@ -0,0 +1,164 @@ +package delete + +import ( + "bytes" + "context" + "fmt" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testEndpoint = "http://localhost:8333" + testAccessKey = "admin" + testSecretKey = "admin" + testRegion = "us-east-1" +) + +func getTestClient(t *testing.T) *s3.Client { + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(testRegion), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider( + testAccessKey, + testSecretKey, + "", + )), + ) + require.NoError(t, err) + + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(testEndpoint) + o.UsePathStyle = true + }) + + return client +} + +func createTestBucket(t *testing.T, client *s3.Client) string { + bucketName := fmt.Sprintf("test-multi-delete-%d", time.Now().UnixNano()) + _, err := client.CreateBucket(context.TODO(), &s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + + _, err = client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucketName), + VersioningConfiguration: &types.VersioningConfiguration{ + Status: types.BucketVersioningStatusEnabled, + }, + }) + require.NoError(t, err) + + return bucketName +} + +func cleanupBucket(t *testing.T, client *s3.Client, bucket string) { + listResp, _ := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + }) + + if listResp != nil { + var objectsToDelete []types.ObjectIdentifier + + for _, version := range listResp.Versions { + objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{ + Key: version.Key, + VersionId: version.VersionId, + }) + } + + for _, marker := range listResp.DeleteMarkers { + objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{ + Key: marker.Key, + VersionId: marker.VersionId, + }) + } + + if len(objectsToDelete) > 0 { + _, _ = client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &types.Delete{ + Objects: objectsToDelete, + Quiet: aws.Bool(false), + }, + }) + } + } + + _, _ = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{ + Bucket: aws.String(bucket), + }) +} + +func TestVersioningMultiObjectDelete(t *testing.T) { + client := getTestClient(t) + bucket := createTestBucket(t, client) + defer cleanupBucket(t, client, bucket) + + key := "key" + numVersions := 2 + var versionIds []string + + for i := 0; i < numVersions; i++ { + content := fmt.Sprintf("content-%d", i) + putResp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader([]byte(content)), + }) + require.NoError(t, err) + require.NotNil(t, putResp.VersionId) + versionIds = append(versionIds, *putResp.VersionId) + } + + assert.Len(t, versionIds, 2) + + var objectsToDelete []types.ObjectIdentifier + for _, vid := range versionIds { + objectsToDelete = append(objectsToDelete, types.ObjectIdentifier{ + Key: aws.String(key), + VersionId: aws.String(vid), + }) + } + + deleteResp, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &types.Delete{ + Objects: objectsToDelete, + }, + }) + require.NoError(t, err) + t.Logf("Delete response: Deleted=%d, Errors=%d", len(deleteResp.Deleted), len(deleteResp.Errors)) + + listResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucket), + }) + require.NoError(t, err) + + if listResp.Versions != nil && len(listResp.Versions) > 0 { + t.Errorf("FAIL: Expected no versions, but found %d versions:", len(listResp.Versions)) + for _, v := range listResp.Versions { + t.Logf(" - Key=%s, VersionId=%s, IsLatest=%v", *v.Key, *v.VersionId, v.IsLatest) + } + } else { + t.Logf("PASS: Versions correctly deleted") + } + assert.Nil(t, listResp.Versions, "Expected no versions after deletion") + + deleteResp2, err := client.DeleteObjects(context.TODO(), &s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &types.Delete{ + Objects: objectsToDelete, + }, + }) + require.NoError(t, err) + assert.Empty(t, deleteResp2.Errors, "Idempotent delete should not return errors") +} diff --git a/test/s3/filer_group/Makefile b/test/s3/filer_group/Makefile index df243d2b0..21d5c050a 100644 --- a/test/s3/filer_group/Makefile +++ b/test/s3/filer_group/Makefile @@ -16,6 +16,8 @@ TEST_PATTERN := Test # Filer group configuration FILER_GROUP := testgroup +SERVER_DIR := ./test-volume-data +S3_CONFIG := ../../../docker/compose/s3.json # Default target help: @@ -44,7 +46,7 @@ build-weed: @echo "Building SeaweedFS binary..." @cd ../../../weed && go build -o weed_binary . @chmod +x $(WEED_BINARY) - @echo "✅ SeaweedFS binary built at $(WEED_BINARY)" + @echo "OK SeaweedFS binary built at $(WEED_BINARY)" check-deps: build-weed @echo "Checking dependencies..." @@ -53,51 +55,40 @@ check-deps: build-weed @test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1) @go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1) @go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1) - @echo "✅ All dependencies are available" + @echo "OK All dependencies are available" # Start SeaweedFS server with filer group configured start-server: check-deps @echo "Starting SeaweedFS server with filer group: $(FILER_GROUP)..." @rm -f weed-server.pid - @mkdir -p ./test-volume-data + @mkdir -p $(SERVER_DIR) @if netstat -tlnp 2>/dev/null | grep $(S3_PORT) >/dev/null; then \ - echo "⚠️ Port $(S3_PORT) is already in use"; \ + echo "WARNING: Port $(S3_PORT) is already in use"; \ exit 1; \ fi @echo "Launching SeaweedFS server with filer group $(FILER_GROUP)..." - @$(WEED_BINARY) server \ + @export AWS_ACCESS_KEY_ID=some_access_key1 && \ + export AWS_SECRET_ACCESS_KEY=some_secret_key1 && \ + $(WEED_BINARY) mini \ -debug \ - -s3 \ + -dir=$(SERVER_DIR) \ -s3.port=$(S3_PORT) \ - -s3.allowDeleteBucketNotEmpty=true \ - -s3.config=../../../docker/compose/s3.json \ - -filer \ - -filer.maxMB=64 \ + -s3.config=$(S3_CONFIG) \ -filer.filerGroup=$(FILER_GROUP) \ - -master.volumeSizeLimitMB=50 \ - -volume.max=100 \ - -dir=./test-volume-data \ - -volume.preStopSeconds=1 \ - -metricsPort=9325 \ - > weed-test.log 2>&1 & echo $$! > weed-server.pid - @echo "Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')" - @echo "Waiting for server to start (up to 90 seconds)..." - @for i in $$(seq 1 90); do \ - if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \ - echo "✅ SeaweedFS server started successfully on port $(S3_PORT) with filer group $(FILER_GROUP)"; \ + > weed-server.log 2>&1 & \ + echo $$! > weed-server.pid + + @echo "Waiting for S3 server to be ready..." + @for i in $$(seq 1 30); do \ + if echo | nc -z localhost $(S3_PORT); then \ + echo "S3 server is ready!"; \ exit 0; \ fi; \ - if [ $$i -eq 30 ]; then \ - echo "⚠️ Server taking longer than expected (30s), checking logs..."; \ - if [ -f weed-test.log ]; then \ - tail -20 weed-test.log; \ - fi; \ - fi; \ sleep 1; \ done; \ - echo "❌ Server failed to start within 90 seconds"; \ - if [ -f weed-test.log ]; then \ - cat weed-test.log; \ + echo "❌ Server failed to start within 30 seconds"; \ + if [ -f weed-server.log ]; then \ + cat weed-server.log; \ fi; \ exit 1 @@ -126,9 +117,9 @@ stop-server: # Show server logs logs: - @if test -f weed-test.log; then \ + @if test -f weed-server.log; then \ echo "=== SeaweedFS Server Logs ==="; \ - tail -f weed-test.log; \ + tail -f weed-server.log; \ else \ echo "No log file found. Server may not be running."; \ fi @@ -146,7 +137,7 @@ test-with-server: start-server @echo "Test pattern: $(TEST_PATTERN)" @echo "Test timeout: $(TEST_TIMEOUT)" @trap "$(MAKE) stop-server" EXIT; \ - $(MAKE) test || (echo "❌ Tests failed, showing server logs:" && echo "=== Last 50 lines of server logs ===" && tail -50 weed-test.log && echo "=== End of server logs ===" && exit 1) + $(MAKE) test || (echo "❌ Tests failed, showing server logs:" && echo "=== Last 50 lines of server logs ===" && tail -50 weed-server.log && echo "=== End of server logs ===" && exit 1) @$(MAKE) stop-server @echo "✅ Tests completed and server stopped" @@ -154,7 +145,7 @@ test-with-server: start-server clean: @echo "Cleaning up test artifacts..." @$(MAKE) stop-server - @rm -f weed-test*.log weed-server.pid + @rm -f weed-server.log weed-test*.log weed-server.pid @rm -rf test-volume-data/ @go clean -testcache @echo "✅ Cleanup completed" diff --git a/test/s3/filer_group/test_config.json b/test/s3/filer_group/test_config.json index 34a4e5d66..05e5c6912 100644 --- a/test/s3/filer_group/test_config.json +++ b/test/s3/filer_group/test_config.json @@ -1,6 +1,5 @@ { "s3_endpoint": "http://localhost:8333", - "master_address": "localhost:9333", "access_key": "some_access_key1", "secret_key": "some_secret_key1", "region": "us-east-1", diff --git a/test/s3/iam/Makefile b/test/s3/iam/Makefile index 7a3f8f950..5113b6b57 100644 --- a/test/s3/iam/Makefile +++ b/test/s3/iam/Makefile @@ -19,6 +19,7 @@ MASTER_PID_FILE = /tmp/weed-master.pid VOLUME_PID_FILE = /tmp/weed-volume.pid FILER_PID_FILE = /tmp/weed-filer.pid S3_PID_FILE = /tmp/weed-s3.pid +MINI_PID_FILE = /tmp/weed-mini.pid help: ## Show this help message @echo "SeaweedFS S3 IAM Integration Tests" @@ -49,80 +50,54 @@ test: clean setup start-services run-tests stop-services ## Run complete IAM int test-quick: run-tests ## Run tests assuming services are already running run-tests: ## Execute the Go tests - @echo "🧪 Running S3 IAM Integration Tests..." + @echo "Running S3 IAM Integration Tests..." go test -v -timeout $(TEST_TIMEOUT) ./... setup: ## Setup test environment - @echo "🔧 Setting up test environment..." + @echo "Setting up test environment..." @mkdir -p test-volume-data/filerldb2 @mkdir -p test-volume-data/m9333 start-services: ## Start SeaweedFS services for testing - @echo "🚀 Starting SeaweedFS services..." - @echo "Starting master server..." - @$(WEED_BINARY) master -port=$(MASTER_PORT) \ - -mdir=test-volume-data/m9333 \ - -peers=none > weed-master.log 2>&1 & \ - echo $$! > $(MASTER_PID_FILE) - - @echo "Waiting for master server to be ready..." - @timeout 60 bash -c 'until curl -s http://localhost:$(MASTER_PORT)/cluster/status > /dev/null 2>&1; do echo "Waiting for master server..."; sleep 2; done' || (echo "❌ Master failed to start, checking logs..." && tail -20 weed-master.log && exit 1) - @echo "✅ Master server is ready" - - @echo "Starting volume server..." - @$(WEED_BINARY) volume -port=$(VOLUME_PORT) \ - -ip=localhost \ - -dataCenter=dc1 -rack=rack1 \ + @echo "Starting SeaweedFS services using weed mini..." + @mkdir -p test-volume-data + @$(WEED_BINARY) mini \ -dir=test-volume-data \ - -max=100 \ - -master=localhost:$(MASTER_PORT) > weed-volume.log 2>&1 & \ - echo $$! > $(VOLUME_PID_FILE) + -s3.port=$(S3_PORT) \ + -s3.config=test_config.json \ + -s3.iam.config=$(CURDIR)/iam_config.json \ + > weed-mini.log 2>&1 & \ + echo $$! > $(MINI_PID_FILE) - @echo "Waiting for volume server to be ready..." - @timeout 60 bash -c 'until curl -s http://localhost:$(VOLUME_PORT)/status > /dev/null 2>&1; do echo "Waiting for volume server..."; sleep 2; done' || (echo "❌ Volume server failed to start, checking logs..." && tail -20 weed-volume.log && exit 1) - @echo "✅ Volume server is ready" - - @echo "Starting filer server..." - @$(WEED_BINARY) filer -port=$(FILER_PORT) \ - -defaultStoreDir=test-volume-data/filerldb2 \ - -master=localhost:$(MASTER_PORT) > weed-filer.log 2>&1 & \ - echo $$! > $(FILER_PID_FILE) - - @echo "Waiting for filer server to be ready..." - @timeout 60 bash -c 'until curl -s http://localhost:$(FILER_PORT)/status > /dev/null 2>&1; do echo "Waiting for filer server..."; sleep 2; done' || (echo "❌ Filer failed to start, checking logs..." && tail -20 weed-filer.log && exit 1) - @echo "✅ Filer server is ready" - - @echo "Starting S3 API server with IAM..." - @$(WEED_BINARY) -v=3 s3 -port=$(S3_PORT) \ - -filer=localhost:$(FILER_PORT) \ - -config=test_config.json \ - -iam.config=$(CURDIR)/iam_config.json > weed-s3.log 2>&1 & \ - echo $$! > $(S3_PID_FILE) - - @echo "Waiting for S3 API server to be ready..." - @timeout 60 bash -c 'until curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1; do echo "Waiting for S3 API server..."; sleep 2; done' || (echo "❌ S3 API failed to start, checking logs..." && tail -20 weed-s3.log && exit 1) - @echo "✅ S3 API server is ready" - - @echo "✅ All services started and ready" + @echo "Waiting for services to be ready..." + @$(MAKE) wait-for-services + @echo "OK All services started and ready" wait-for-services: ## Wait for all services to be ready - @echo "⏳ Waiting for services to be ready..." - @echo "Checking master server..." - @timeout 30 bash -c 'until curl -s http://localhost:$(MASTER_PORT)/cluster/status > /dev/null; do sleep 1; done' || (echo "❌ Master failed to start" && exit 1) - - @echo "Checking filer server..." - @timeout 30 bash -c 'until curl -s http://localhost:$(FILER_PORT)/status > /dev/null; do sleep 1; done' || (echo "❌ Filer failed to start" && exit 1) - + @echo "Waiting for services to be ready..." @echo "Checking S3 API server..." - @timeout 30 bash -c 'until curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1; do sleep 1; done' || (echo "❌ S3 API failed to start" && exit 1) + @for i in $$(seq 1 30); do \ + if curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1; then \ + echo "OK S3 API server is ready"; \ + exit 0; \ + fi; \ + sleep 1; \ + done; \ + echo "ERROR S3 API failed to start"; \ + exit 1 @echo "Pre-allocating volumes for concurrent operations..." - @curl -s "http://localhost:$(MASTER_PORT)/vol/grow?collection=default&count=10&replication=000" > /dev/null || echo "⚠️ Volume pre-allocation failed, but continuing..." + @curl -s "http://localhost:$(MASTER_PORT)/vol/grow?collection=default&count=10&replication=000" > /dev/null || echo "WARNING Volume pre-allocation failed, but continuing..." @sleep 3 - @echo "✅ All services are ready" + @echo "OK All services are ready" stop-services: ## Stop all SeaweedFS services - @echo "🛑 Stopping SeaweedFS services..." + @echo "Stopping SeaweedFS services..." + @if [ -f $(MINI_PID_FILE) ]; then \ + echo "Stopping weed mini..."; \ + kill $$(cat $(MINI_PID_FILE)) 2>/dev/null || true; \ + rm -f $(MINI_PID_FILE); \ + fi @if [ -f $(S3_PID_FILE) ]; then \ echo "Stopping S3 API server..."; \ kill $$(cat $(S3_PID_FILE)) 2>/dev/null || true; \ @@ -143,17 +118,17 @@ stop-services: ## Stop all SeaweedFS services kill $$(cat $(MASTER_PID_FILE)) 2>/dev/null || true; \ rm -f $(MASTER_PID_FILE); \ fi - @echo "✅ All services stopped" + @echo "OK All services stopped" clean: stop-services ## Clean up test environment - @echo "🧹 Cleaning up test environment..." + @echo "Cleaning up test environment..." @rm -rf test-volume-data @rm -f weed-*.log @rm -f *.test - @echo "✅ Cleanup complete" + @echo "Cleanup complete" logs: ## Show service logs - @echo "📋 Service Logs:" + @echo "Service Logs:" @echo "=== Master Log ===" @tail -20 weed-master.log 2>/dev/null || echo "No master log" @echo "" @@ -167,7 +142,7 @@ logs: ## Show service logs @tail -20 weed-s3.log 2>/dev/null || echo "No S3 log" status: ## Check service status - @echo "📊 Service Status:" + @echo "Service Status:" @echo -n "Master: "; curl -s http://localhost:$(MASTER_PORT)/cluster/status > /dev/null 2>&1 && echo "✅ Running" || echo "❌ Not running" @echo -n "Filer: "; curl -s http://localhost:$(FILER_PORT)/status > /dev/null 2>&1 && echo "✅ Running" || echo "❌ Not running" @echo -n "S3 API: "; curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1 && echo "✅ Running" || echo "❌ Not running" diff --git a/test/s3/iam/s3_service_account_security_test.go b/test/s3/iam/s3_service_account_security_test.go new file mode 100644 index 000000000..a01ddbcdb --- /dev/null +++ b/test/s3/iam/s3_service_account_security_test.go @@ -0,0 +1,431 @@ +package iam + +// Integration tests for SeaweedFS service accounts. +// These tests ensure comprehensive coverage of service account functionality +// including security, access control, and expiration. + +import ( + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestServiceAccountS3Access verifies that service accounts can actually +// perform S3 operations using their credentials. +func TestServiceAccountS3Access(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + if !isSeaweedFSRunning(t) { + t.Skip("SeaweedFS is not running at", TestIAMEndpoint) + } + + // Setup: Create a parent user + parentUserName := fmt.Sprintf("s3access-test-%d", time.Now().UnixNano()) + + // Create parent user + resp, err := callIAMAPI(t, "CreateUser", url.Values{ + "UserName": {parentUserName}, + }) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode, "Failed to create parent user") + + defer func() { + // Cleanup: delete parent user + callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}}) + }() + + // Create service account for the parent user + createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{ + "ParentUser": {parentUserName}, + "Description": {"S3 Access Test Service Account"}, + }) + require.NoError(t, err) + defer createResp.Body.Close() + require.Equal(t, http.StatusOK, createResp.StatusCode, "Failed to create service account") + + body, err := io.ReadAll(createResp.Body) + require.NoError(t, err) + + var saResp CreateServiceAccountResponse + err = xml.Unmarshal(body, &saResp) + require.NoError(t, err, "Failed to parse CreateServiceAccount response: %s", string(body)) + + accessKeyId := saResp.CreateServiceAccountResult.ServiceAccount.AccessKeyId + secretAccessKey := saResp.CreateServiceAccountResult.ServiceAccount.SecretAccessKey + saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId + + require.NotEmpty(t, accessKeyId, "AccessKeyId should not be empty") + require.NotEmpty(t, secretAccessKey, "SecretAccessKey should not be empty") + + defer func() { + // Cleanup: delete service account + callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}}) + }() + + t.Run("list_buckets_with_sa_credentials", func(t *testing.T) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-1"), + Endpoint: aws.String(TestIAMEndpoint), + Credentials: credentials.NewStaticCredentials( + accessKeyId, + secretAccessKey, + "", + ), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + }) + require.NoError(t, err) + + s3Client := s3.New(sess) + _, err = s3Client.ListBuckets(&s3.ListBucketsInput{}) + + // We don't necessarily expect success (depends on permissions), + // but we should NOT get InvalidAccessKeyId or SignatureDoesNotMatch + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + assert.NotEqual(t, "InvalidAccessKeyId", aerr.Code(), + "Service account credentials should be recognized") + assert.NotEqual(t, "SignatureDoesNotMatch", aerr.Code(), + "Service account signature should be valid") + } + } + }) + + t.Run("create_bucket_with_sa_credentials", func(t *testing.T) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-1"), + Endpoint: aws.String(TestIAMEndpoint), + Credentials: credentials.NewStaticCredentials( + accessKeyId, + secretAccessKey, + "", + ), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + }) + require.NoError(t, err) + + s3Client := s3.New(sess) + bucketName := fmt.Sprintf("sa-test-bucket-%d", time.Now().UnixNano()) + + _, err = s3Client.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucketName), + }) + + // Check that we get a proper response (success or AccessDenied based on policy) + // but NOT InvalidAccessKeyId + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + assert.NotEqual(t, "InvalidAccessKeyId", aerr.Code(), + "Service account credentials should be recognized") + assert.NotEqual(t, "SignatureDoesNotMatch", aerr.Code(), + "Service account signature should be valid") + } + } else { + // Cleanup if bucket was created + defer s3Client.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucketName), + }) + } + }) +} + +// TestServiceAccountExpiration verifies that expired service accounts +// are properly rejected. +func TestServiceAccountExpiration(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + if !isSeaweedFSRunning(t) { + t.Skip("SeaweedFS is not running at", TestIAMEndpoint) + } + + // Setup: Create a parent user + parentUserName := fmt.Sprintf("expiry-test-%d", time.Now().UnixNano()) + + resp, err := callIAMAPI(t, "CreateUser", url.Values{ + "UserName": {parentUserName}, + }) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + defer func() { + callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}}) + }() + + t.Run("reject_past_expiration", func(t *testing.T) { + // Try to create a service account with expiration in the past + pastExpiration := time.Now().Add(-1 * time.Hour).Unix() + + createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{ + "ParentUser": {parentUserName}, + "Description": {"Should fail - past expiration"}, + "Expiration": {strconv.FormatInt(pastExpiration, 10)}, + }) + require.NoError(t, err) + defer createResp.Body.Close() + + // Should fail because expiration is in the past + assert.NotEqual(t, http.StatusOK, createResp.StatusCode, + "Creating service account with past expiration should fail") + }) + + t.Run("accept_future_expiration", func(t *testing.T) { + // Create a service account with expiration in the future + futureExpiration := time.Now().Add(24 * time.Hour).Unix() + + createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{ + "ParentUser": {parentUserName}, + "Description": {"Should succeed - future expiration"}, + "Expiration": {strconv.FormatInt(futureExpiration, 10)}, + }) + require.NoError(t, err) + defer createResp.Body.Close() + + assert.Equal(t, http.StatusOK, createResp.StatusCode, + "Creating service account with future expiration should succeed") + + // Parse response to get service account ID for cleanup + if createResp.StatusCode == http.StatusOK { + body, err := io.ReadAll(createResp.Body) + require.NoError(t, err) + + var saResp CreateServiceAccountResponse + require.NoError(t, xml.Unmarshal(body, &saResp)) + + saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId + if saId != "" { + t.Cleanup(func() { + callIAMAPI(t, "DeleteServiceAccount", url.Values{ + "ServiceAccountId": {saId}, + }) + }) + } + } + }) + + t.Run("reject_past_expiration_on_update", func(t *testing.T) { + // Create a valid service account first + futureExpiration := time.Now().Add(24 * time.Hour).Unix() + + createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{ + "ParentUser": {parentUserName}, + "Description": {"For update test"}, + "Expiration": {strconv.FormatInt(futureExpiration, 10)}, + }) + require.NoError(t, err) + defer createResp.Body.Close() + require.Equal(t, http.StatusOK, createResp.StatusCode) + + body, err := io.ReadAll(createResp.Body) + require.NoError(t, err) + + var saResp CreateServiceAccountResponse + err = xml.Unmarshal(body, &saResp) + require.NoError(t, err) + + saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId + require.NotEmpty(t, saId) + + defer func() { + callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}}) + }() + + // Try to update with past expiration + pastExpiration := time.Now().Add(-1 * time.Hour).Unix() + + updateResp, err := callIAMAPI(t, "UpdateServiceAccount", url.Values{ + "ServiceAccountId": {saId}, + "Expiration": {strconv.FormatInt(pastExpiration, 10)}, + }) + require.NoError(t, err) + defer updateResp.Body.Close() + + // Should fail because expiration is in the past + assert.NotEqual(t, http.StatusOK, updateResp.StatusCode, + "Updating service account with past expiration should fail") + }) +} + +// TestServiceAccountInheritedPermissions verifies that service accounts +// inherit their parent user's permissions. +// This is a key security test - SAs should not have MORE permissions than parent. +func TestServiceAccountInheritedPermissions(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + if !isSeaweedFSRunning(t) { + t.Skip("SeaweedFS is not running at", TestIAMEndpoint) + } + + // Setup: Create a parent user + parentUserName := fmt.Sprintf("inherit-test-%d", time.Now().UnixNano()) + + resp, err := callIAMAPI(t, "CreateUser", url.Values{ + "UserName": {parentUserName}, + }) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + defer func() { + callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}}) + }() + + // Create service account + createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{ + "ParentUser": {parentUserName}, + "Description": {"Permissions inheritance test"}, + }) + require.NoError(t, err) + defer createResp.Body.Close() + require.Equal(t, http.StatusOK, createResp.StatusCode) + + body, err := io.ReadAll(createResp.Body) + require.NoError(t, err) + + var saResp CreateServiceAccountResponse + err = xml.Unmarshal(body, &saResp) + require.NoError(t, err) + + saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId + require.NotEmpty(t, saId) + + defer func() { + callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}}) + }() + + t.Run("service_account_linked_to_parent", func(t *testing.T) { + // Verify the service account is correctly linked to the parent + getResp, err := callIAMAPI(t, "GetServiceAccount", url.Values{ + "ServiceAccountId": {saId}, + }) + require.NoError(t, err) + defer getResp.Body.Close() + + body, err := io.ReadAll(getResp.Body) + require.NoError(t, err) + + var result GetServiceAccountResponse + err = xml.Unmarshal(body, &result) + require.NoError(t, err, "Failed to parse response: %s", string(body)) + + assert.Equal(t, parentUserName, result.GetServiceAccountResult.ServiceAccount.ParentUser, + "Service account should be linked to correct parent user") + }) + + t.Run("list_shows_correct_parent", func(t *testing.T) { + // List service accounts filtered by parent + listResp, err := callIAMAPI(t, "ListServiceAccounts", url.Values{ + "ParentUser": {parentUserName}, + }) + require.NoError(t, err) + defer listResp.Body.Close() + + body, err := io.ReadAll(listResp.Body) + require.NoError(t, err) + + var listResult ListServiceAccountsResponse + err = xml.Unmarshal(body, &listResult) + require.NoError(t, err) + + // Should find at least one service account for this parent + found := false + for _, sa := range listResult.ListServiceAccountsResult.ServiceAccounts { + if sa.ServiceAccountId == saId { + found = true + assert.Equal(t, parentUserName, sa.ParentUser) + break + } + } + assert.True(t, found, "Service account should appear in list filtered by parent") + }) +} + +// TestServiceAccountAccessKeyFormat verifies that service account access keys +// follow the correct AWS format. +func TestServiceAccountAccessKeyFormat(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + if !isSeaweedFSRunning(t) { + t.Skip("SeaweedFS is not running at", TestIAMEndpoint) + } + + // Setup: Create a parent user + parentUserName := fmt.Sprintf("keyformat-test-%d", time.Now().UnixNano()) + + resp, err := callIAMAPI(t, "CreateUser", url.Values{ + "UserName": {parentUserName}, + }) + require.NoError(t, err) + resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + defer func() { + callIAMAPI(t, "DeleteUser", url.Values{"UserName": {parentUserName}}) + }() + + createResp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{ + "ParentUser": {parentUserName}, + "Description": {"Key format test"}, + }) + require.NoError(t, err) + defer createResp.Body.Close() + require.Equal(t, http.StatusOK, createResp.StatusCode) + + body, err := io.ReadAll(createResp.Body) + require.NoError(t, err) + + var saResp CreateServiceAccountResponse + err = xml.Unmarshal(body, &saResp) + require.NoError(t, err) + + accessKeyId := saResp.CreateServiceAccountResult.ServiceAccount.AccessKeyId + secretAccessKey := saResp.CreateServiceAccountResult.ServiceAccount.SecretAccessKey + saId := saResp.CreateServiceAccountResult.ServiceAccount.ServiceAccountId + + defer func() { + callIAMAPI(t, "DeleteServiceAccount", url.Values{"ServiceAccountId": {saId}}) + }() + + t.Run("access_key_has_correct_prefix", func(t *testing.T) { + // Service account access keys should start with ABIA + assert.True(t, len(accessKeyId) >= 4, + "Access key should be at least 4 characters") + assert.Equal(t, "ABIA", accessKeyId[:4], + "Service account access key should start with ABIA prefix") + }) + + t.Run("access_key_has_correct_length", func(t *testing.T) { + // AWS access keys are 20 characters + assert.Equal(t, 20, len(accessKeyId), + "Access key should be exactly 20 characters (AWS standard)") + }) + + t.Run("secret_key_has_correct_length", func(t *testing.T) { + // AWS secret keys are 40 characters + assert.Equal(t, 40, len(secretAccessKey), + "Secret key should be exactly 40 characters (AWS standard)") + }) +} diff --git a/test/s3/iam/s3_service_account_test.go b/test/s3/iam/s3_service_account_test.go new file mode 100644 index 000000000..a4e5ab68b --- /dev/null +++ b/test/s3/iam/s3_service_account_test.go @@ -0,0 +1,363 @@ +package iam + +import ( + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Service Account API test constants +const ( + TestIAMEndpoint = "http://localhost:8333" +) + +// ServiceAccountInfo represents the response structure for service account operations +type ServiceAccountInfo struct { + ServiceAccountId string `xml:"ServiceAccountId"` + ParentUser string `xml:"ParentUser"` + Description string `xml:"Description,omitempty"` + AccessKeyId string `xml:"AccessKeyId"` + SecretAccessKey string `xml:"SecretAccessKey,omitempty"` + Status string `xml:"Status"` + Expiration string `xml:"Expiration,omitempty"` + CreateDate string `xml:"CreateDate"` +} + +// CreateServiceAccountResponse represents the response for CreateServiceAccount +type CreateServiceAccountResponse struct { + XMLName xml.Name `xml:"CreateServiceAccountResponse"` + CreateServiceAccountResult struct { + ServiceAccount ServiceAccountInfo `xml:"ServiceAccount"` + } `xml:"CreateServiceAccountResult"` +} + +// ListServiceAccountsResponse represents the response for ListServiceAccounts +type ListServiceAccountsResponse struct { + XMLName xml.Name `xml:"ListServiceAccountsResponse"` + ListServiceAccountsResult struct { + ServiceAccounts []ServiceAccountInfo `xml:"ServiceAccounts>member"` + IsTruncated bool `xml:"IsTruncated"` + } `xml:"ListServiceAccountsResult"` +} + +// GetServiceAccountResponse represents the response for GetServiceAccount +type GetServiceAccountResponse struct { + XMLName xml.Name `xml:"GetServiceAccountResponse"` + GetServiceAccountResult struct { + ServiceAccount ServiceAccountInfo `xml:"ServiceAccount"` + } `xml:"GetServiceAccountResult"` +} + +// TestServiceAccountLifecycle tests the complete lifecycle of service accounts +// This is a high-value test covering Create, Get, List, Update, Delete operations +func TestServiceAccountLifecycle(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + // Check if SeaweedFS is running + if !isSeaweedFSRunning(t) { + t.Skip("SeaweedFS is not running at", TestIAMEndpoint) + } + + // First, ensure the parent user exists + parentUserName := fmt.Sprintf("testuser-%d", time.Now().UnixNano()) + + t.Run("create_parent_user", func(t *testing.T) { + resp, err := callIAMAPI(t, "CreateUser", url.Values{ + "UserName": {parentUserName}, + }) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode, "CreateUser should succeed") + }) + + // Store service account IDs for cleanup + var createdServiceAccounts []string + + defer func() { + // Cleanup: delete all service accounts first, then parent user + for _, saId := range createdServiceAccounts { + resp, _ := callIAMAPI(t, "DeleteServiceAccount", url.Values{ + "ServiceAccountId": {saId}, + }) + if resp != nil { + resp.Body.Close() + } + } + // Now delete the parent user + resp, _ := callIAMAPI(t, "DeleteUser", url.Values{ + "UserName": {parentUserName}, + }) + if resp != nil { + resp.Body.Close() + } + }() + + var createdSAId string + var createdAccessKeyId string + var createdSecretAccessKey string + + t.Run("create_service_account", func(t *testing.T) { + resp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{ + "ParentUser": {parentUserName}, + "Description": {"Test service account for CI/CD"}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode, "CreateServiceAccount should succeed") + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + var createResp CreateServiceAccountResponse + err = xml.Unmarshal(body, &createResp) + require.NoError(t, err) + + sa := createResp.CreateServiceAccountResult.ServiceAccount + createdSAId = sa.ServiceAccountId + createdAccessKeyId = sa.AccessKeyId + createdSecretAccessKey = sa.SecretAccessKey + + // Add to cleanup list + createdServiceAccounts = append(createdServiceAccounts, createdSAId) + + assert.NotEmpty(t, createdSAId, "ServiceAccountId should not be empty") + assert.Equal(t, parentUserName, sa.ParentUser, "ParentUser should match") + assert.Equal(t, "Test service account for CI/CD", sa.Description) + assert.Equal(t, "Active", sa.Status) + assert.NotEmpty(t, sa.AccessKeyId, "AccessKeyId should not be empty") + assert.NotEmpty(t, sa.SecretAccessKey, "SecretAccessKey should be returned on create") + assert.True(t, strings.HasPrefix(sa.AccessKeyId, "ABIA"), + "Service account AccessKeyId should have ABIA prefix") + + t.Logf("Created service account: ID=%s, AccessKeyId=%s", createdSAId, createdAccessKeyId) + }) + + t.Run("get_service_account", func(t *testing.T) { + require.NotEmpty(t, createdSAId, "Service account should have been created") + + resp, err := callIAMAPI(t, "GetServiceAccount", url.Values{ + "ServiceAccountId": {createdSAId}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + var getResp GetServiceAccountResponse + err = xml.Unmarshal(body, &getResp) + require.NoError(t, err) + + sa := getResp.GetServiceAccountResult.ServiceAccount + assert.Equal(t, createdSAId, sa.ServiceAccountId) + assert.Equal(t, parentUserName, sa.ParentUser) + assert.Empty(t, sa.SecretAccessKey, "SecretAccessKey should not be returned on Get") + }) + + t.Run("list_service_accounts", func(t *testing.T) { + resp, err := callIAMAPI(t, "ListServiceAccounts", url.Values{ + "ParentUser": {parentUserName}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + var listResp ListServiceAccountsResponse + err = xml.Unmarshal(body, &listResp) + require.NoError(t, err) + + assert.GreaterOrEqual(t, len(listResp.ListServiceAccountsResult.ServiceAccounts), 1, + "Should have at least one service account for the parent user") + }) + + t.Run("update_service_account_status", func(t *testing.T) { + require.NotEmpty(t, createdSAId) + + // Disable the service account + resp, err := callIAMAPI(t, "UpdateServiceAccount", url.Values{ + "ServiceAccountId": {createdSAId}, + "Status": {"Inactive"}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // Verify it's now inactive + getResp, err := callIAMAPI(t, "GetServiceAccount", url.Values{ + "ServiceAccountId": {createdSAId}, + }) + require.NoError(t, err) + defer getResp.Body.Close() + + body, err := io.ReadAll(getResp.Body) + require.NoError(t, err) + + var result GetServiceAccountResponse + err = xml.Unmarshal(body, &result) + require.NoError(t, err, "Failed to parse response: %s", string(body)) + + assert.Equal(t, "Inactive", result.GetServiceAccountResult.ServiceAccount.Status) + }) + + // Test that credentials could be used (verify they work with AWS SDK) + // This must run BEFORE delete_service_account to use valid credentials + t.Run("use_service_account_credentials", func(t *testing.T) { + require.NotEmpty(t, createdAccessKeyId) + require.NotEmpty(t, createdSecretAccessKey) + + sess, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-1"), + Endpoint: aws.String(TestIAMEndpoint), // IAM and S3 usually on same port in mini-seaweed + Credentials: credentials.NewStaticCredentials( + createdAccessKeyId, + createdSecretAccessKey, + "", + ), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + }) + require.NoError(t, err) + + s3Client := s3.New(sess) + _, err = s3Client.ListBuckets(&s3.ListBucketsInput{}) + + // Note: we don't necessarily expect success if no buckets/permissions + // but we expect it not to fail with "InvalidAccessKeyId" or "SignatureDoesNotMatch" + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + assert.NotEqual(t, "InvalidAccessKeyId", aerr.Code(), "Credentials should be valid") + assert.NotEqual(t, "SignatureDoesNotMatch", aerr.Code(), "Signature should be valid") + } + } + }) + + t.Run("delete_service_account", func(t *testing.T) { + require.NotEmpty(t, createdSAId) + + resp, err := callIAMAPI(t, "DeleteServiceAccount", url.Values{ + "ServiceAccountId": {createdSAId}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // Verify it no longer exists + getResp, err := callIAMAPI(t, "GetServiceAccount", url.Values{ + "ServiceAccountId": {createdSAId}, + }) + require.NoError(t, err) + defer getResp.Body.Close() + + // Should return an error (not found) + assert.NotEqual(t, http.StatusOK, getResp.StatusCode, + "GetServiceAccount should fail after deletion") + }) + +} + +// TestServiceAccountValidation tests validation of service account operations +func TestServiceAccountValidation(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + if !isSeaweedFSRunning(t) { + t.Skip("SeaweedFS is not running at", TestIAMEndpoint) + } + + t.Run("create_without_parent_user", func(t *testing.T) { + resp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{ + "Description": {"Test without parent"}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.NotEqual(t, http.StatusOK, resp.StatusCode, + "CreateServiceAccount without ParentUser should fail") + }) + + t.Run("create_with_nonexistent_parent", func(t *testing.T) { + resp, err := callIAMAPI(t, "CreateServiceAccount", url.Values{ + "ParentUser": {"nonexistent-user-12345"}, + "Description": {"Test with nonexistent parent"}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.NotEqual(t, http.StatusOK, resp.StatusCode, + "CreateServiceAccount with nonexistent parent should fail") + }) + + t.Run("get_nonexistent_service_account", func(t *testing.T) { + resp, err := callIAMAPI(t, "GetServiceAccount", url.Values{ + "ServiceAccountId": {"sa-NONEXISTENT123"}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.NotEqual(t, http.StatusOK, resp.StatusCode, + "GetServiceAccount for nonexistent ID should fail") + }) + + t.Run("delete_nonexistent_service_account", func(t *testing.T) { + resp, err := callIAMAPI(t, "DeleteServiceAccount", url.Values{ + "ServiceAccountId": {"sa-NONEXISTENT123"}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.NotEqual(t, http.StatusOK, resp.StatusCode, + "DeleteServiceAccount for nonexistent ID should fail") + }) +} + +// callIAMAPI is a helper to make IAM API calls +func callIAMAPI(t *testing.T, action string, params url.Values) (*http.Response, error) { + params.Set("Action", action) + + req, err := http.NewRequest(http.MethodPost, TestIAMEndpoint+"/", + strings.NewReader(params.Encode())) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + client := &http.Client{Timeout: 30 * time.Second} + return client.Do(req) +} + +// isSeaweedFSRunning checks if SeaweedFS S3 API is running +func isSeaweedFSRunning(t *testing.T) bool { + client := &http.Client{Timeout: 2 * time.Second} + resp, err := client.Get(TestIAMEndpoint + "/status") + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK +} diff --git a/test/s3/iam/s3_sts_test.go b/test/s3/iam/s3_sts_test.go new file mode 100644 index 000000000..84daf27a9 --- /dev/null +++ b/test/s3/iam/s3_sts_test.go @@ -0,0 +1,260 @@ +package iam + +import ( + "encoding/xml" + "io" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// STS API test constants +const ( + TestSTSEndpoint = "http://localhost:8333" +) + +// AssumeRoleWithWebIdentityResponse represents the STS response +type AssumeRoleWithWebIdentityTestResponse struct { + XMLName xml.Name `xml:"AssumeRoleWithWebIdentityResponse"` + Result struct { + Credentials struct { + AccessKeyId string `xml:"AccessKeyId"` + SecretAccessKey string `xml:"SecretAccessKey"` + SessionToken string `xml:"SessionToken"` + Expiration string `xml:"Expiration"` + } `xml:"Credentials"` + SubjectFromWebIdentityToken string `xml:"SubjectFromWebIdentityToken,omitempty"` + } `xml:"AssumeRoleWithWebIdentityResult"` +} + +// STSErrorResponse represents an STS error response +type STSErrorTestResponse struct { + XMLName xml.Name `xml:"ErrorResponse"` + Error struct { + Type string `xml:"Type"` + Code string `xml:"Code"` + Message string `xml:"Message"` + } `xml:"Error"` + RequestId string `xml:"RequestId"` +} + +// TestAssumeRoleWithWebIdentityValidation tests input validation for the STS endpoint +func TestAssumeRoleWithWebIdentityValidation(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + if !isSTSEndpointRunning(t) { + t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint) + } + + t.Run("missing_web_identity_token", func(t *testing.T) { + resp, err := callSTSAPI(t, url.Values{ + "Action": {"AssumeRoleWithWebIdentity"}, + "RoleArn": {"arn:aws:iam::role/test-role"}, + "RoleSessionName": {"test-session"}, + // WebIdentityToken is missing + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.NotEqual(t, http.StatusOK, resp.StatusCode, + "Should fail without WebIdentityToken") + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + var errResp STSErrorTestResponse + err = xml.Unmarshal(body, &errResp) + require.NoError(t, err, "Failed to parse error response: %s", string(body)) + assert.Equal(t, "MissingParameter", errResp.Error.Code) + }) + + t.Run("missing_role_arn", func(t *testing.T) { + resp, err := callSTSAPI(t, url.Values{ + "Action": {"AssumeRoleWithWebIdentity"}, + "WebIdentityToken": {"fake-jwt-token"}, + "RoleSessionName": {"test-session"}, + // RoleArn is missing + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.NotEqual(t, http.StatusOK, resp.StatusCode, + "Should fail without RoleArn") + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + var errResp STSErrorTestResponse + err = xml.Unmarshal(body, &errResp) + require.NoError(t, err, "Failed to parse error response: %s", string(body)) + assert.Equal(t, "MissingParameter", errResp.Error.Code) + }) + + t.Run("missing_role_session_name", func(t *testing.T) { + resp, err := callSTSAPI(t, url.Values{ + "Action": {"AssumeRoleWithWebIdentity"}, + "WebIdentityToken": {"fake-jwt-token"}, + "RoleArn": {"arn:aws:iam::role/test-role"}, + // RoleSessionName is missing + }) + require.NoError(t, err) + defer resp.Body.Close() + + assert.NotEqual(t, http.StatusOK, resp.StatusCode, + "Should fail without RoleSessionName") + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + var errResp STSErrorTestResponse + err = xml.Unmarshal(body, &errResp) + require.NoError(t, err, "Failed to parse error response: %s", string(body)) + assert.Equal(t, "MissingParameter", errResp.Error.Code) + }) + + t.Run("invalid_jwt_token", func(t *testing.T) { + resp, err := callSTSAPI(t, url.Values{ + "Action": {"AssumeRoleWithWebIdentity"}, + "WebIdentityToken": {"not-a-valid-jwt-token"}, + "RoleArn": {"arn:aws:iam::role/test-role"}, + "RoleSessionName": {"test-session"}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + // Should fail with AccessDenied since the JWT is invalid + assert.NotEqual(t, http.StatusOK, resp.StatusCode, + "Should fail with invalid JWT token") + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + var errResp STSErrorTestResponse + err = xml.Unmarshal(body, &errResp) + require.NoError(t, err, "Failed to parse error response: %s", string(body)) + assert.Contains(t, []string{"AccessDenied", "InvalidParameterValue"}, errResp.Error.Code) + }) +} + +// TestAssumeRoleWithWebIdentityWithMockJWT tests the STS endpoint with mock JWTs +// This test requires the mock OIDC provider to be configured +func TestAssumeRoleWithWebIdentityWithMockJWT(t *testing.T) { + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + + if !isSTSEndpointRunning(t) { + t.Skip("SeaweedFS STS endpoint is not running at", TestSTSEndpoint) + } + + // Create a test framework to get valid JWT tokens + framework := NewS3IAMTestFramework(t) + defer framework.Cleanup() + + // Generate a test JWT using the framework + testUsername := "sts-test-user" + testRole := "readonly" + + // Try to get a token - use Keycloak if available, otherwise generate a mock JWT + var token string + var err error + if framework.useKeycloak { + token, err = framework.getKeycloakToken(testUsername) + } else { + // Generate a mock JWT token with 1 hour validity + token, err = framework.generateJWTToken(testUsername, testRole, time.Hour) + } + if err != nil { + t.Skipf("Unable to generate test JWT (requires mock OIDC or Keycloak): %v", err) + } + + t.Run("valid_jwt_token", func(t *testing.T) { + resp, err := callSTSAPI(t, url.Values{ + "Action": {"AssumeRoleWithWebIdentity"}, + "WebIdentityToken": {token}, + "RoleArn": {"arn:aws:iam::role/" + testRole}, + "RoleSessionName": {"integration-test-session"}, + }) + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + t.Logf("Response status: %d, body: %s", resp.StatusCode, string(body)) + + // Note: This may still fail if the role/trust policy is not configured + // In that case, we just verify the error is about trust policy, not token validation + if resp.StatusCode != http.StatusOK { + var errResp STSErrorTestResponse + err = xml.Unmarshal(body, &errResp) + require.NoError(t, err, "Failed to parse error response: %s", string(body)) + assert.NotEqual(t, "InvalidParameterValue", errResp.Error.Code, + "Token validation should not fail - error should be about trust policy") + } else { + var stsResp AssumeRoleWithWebIdentityTestResponse + err = xml.Unmarshal(body, &stsResp) + require.NoError(t, err, "Failed to parse response: %s", string(body)) + + creds := stsResp.Result.Credentials + assert.NotEmpty(t, creds.AccessKeyId, "AccessKeyId should not be empty") + assert.NotEmpty(t, creds.SecretAccessKey, "SecretAccessKey should not be empty") + assert.NotEmpty(t, creds.SessionToken, "SessionToken should not be empty") + assert.NotEmpty(t, creds.Expiration, "Expiration should not be empty") + + t.Logf("Successfully obtained temporary credentials: AccessKeyId=%s", creds.AccessKeyId) + } + }) + + t.Run("with_duration_seconds", func(t *testing.T) { + resp, err := callSTSAPI(t, url.Values{ + "Action": {"AssumeRoleWithWebIdentity"}, + "WebIdentityToken": {token}, + "RoleArn": {"arn:aws:iam::role/" + testRole}, + "RoleSessionName": {"integration-test-session"}, + "DurationSeconds": {"3600"}, // 1 hour + }) + require.NoError(t, err) + defer resp.Body.Close() + + // Verify the request is accepted (even if trust policy causes rejection) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + // Should not fail with InvalidParameterValue for DurationSeconds + if resp.StatusCode != http.StatusOK { + var errResp STSErrorTestResponse + err = xml.Unmarshal(body, &errResp) + require.NoError(t, err, "Failed to parse error response: %s", string(body)) + assert.NotContains(t, errResp.Error.Message, "DurationSeconds", + "DurationSeconds parameter should be accepted") + } + }) +} + +// callSTSAPI is a helper to make STS API calls +func callSTSAPI(t *testing.T, params url.Values) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, TestSTSEndpoint+"/", + strings.NewReader(params.Encode())) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + client := &http.Client{Timeout: 30 * time.Second} + return client.Do(req) +} + +// isSTSEndpointRunning checks if SeaweedFS STS endpoint is running +func isSTSEndpointRunning(t *testing.T) bool { + client := &http.Client{Timeout: 2 * time.Second} + resp, err := client.Get(TestSTSEndpoint + "/status") + if err != nil { + return false + } + defer resp.Body.Close() + return resp.StatusCode == http.StatusOK +} diff --git a/test/s3/parquet/CROSS_FILESYSTEM_COMPATIBILITY.md b/test/s3/parquet/CROSS_FILESYSTEM_COMPATIBILITY.md deleted file mode 100644 index 62ef9563d..000000000 --- a/test/s3/parquet/CROSS_FILESYSTEM_COMPATIBILITY.md +++ /dev/null @@ -1,172 +0,0 @@ -# Cross-Filesystem Compatibility Test Results - -## Overview - -This document summarizes the cross-filesystem compatibility testing between **s3fs** and **PyArrow native S3 filesystem** implementations when working with SeaweedFS. - -## Test Purpose - -Verify that Parquet files written using one filesystem implementation (s3fs or PyArrow native S3) can be correctly read using the other implementation, confirming true file format compatibility. - -## Test Methodology - -### Test Matrix - -The test performs the following combinations: - -1. **Write with s3fs → Read with PyArrow native S3** -2. **Write with PyArrow native S3 → Read with s3fs** - -For each direction, the test: -- Creates a sample PyArrow table with multiple data types (int64, string, float64, bool) -- Writes the Parquet file using one filesystem implementation -- Reads the Parquet file using the other filesystem implementation -- Verifies data integrity by comparing: - - Row counts - - Schema equality - - Data contents (after sorting by ID to handle row order differences) - -### File Sizes Tested - -- **Small files**: 5 rows (quick validation) -- **Large files**: 200,000 rows (multi-row-group validation) - -## Test Results - -### ✅ Small Files (5 rows) - -| Write Method | Read Method | Result | Read Function Used | -|--------------|-------------|--------|--------------------| -| s3fs | PyArrow native S3 | ✅ PASS | pq.read_table | -| PyArrow native S3 | s3fs | ✅ PASS | pq.read_table | - -**Status**: **ALL TESTS PASSED** - -### Large Files (200,000 rows) - -Large file testing requires adequate volume capacity in SeaweedFS. When run with default volume settings (50MB max size), tests may encounter capacity issues with the number of large test files created simultaneously. - -**Recommendation**: For large file testing, increase `VOLUME_MAX_SIZE_MB` in the Makefile or run tests with `TEST_QUICK=1` for development/validation purposes. - -## Key Findings - -### ✅ Full Compatibility Confirmed - -**Files written with s3fs and PyArrow native S3 filesystem are fully compatible and can be read by either implementation.** - -This confirms that: - -1. **Identical Parquet Format**: Both s3fs and PyArrow native S3 use the same underlying PyArrow library to generate Parquet files, resulting in identical file formats at the binary level. - -2. **S3 API Compatibility**: SeaweedFS's S3 implementation handles both filesystem backends correctly, with proper: - - Object creation (PutObject) - - Object reading (GetObject) - - Directory handling (implicit directories) - - Multipart uploads (for larger files) - -3. **Metadata Consistency**: File metadata, schemas, and data integrity are preserved across both write and read operations regardless of which filesystem implementation is used. - -## Implementation Details - -### Common Write Path - -Both implementations use PyArrow's `pads.write_dataset()` function: - -```python -# s3fs approach -fs = s3fs.S3FileSystem(...) -pads.write_dataset(table, path, format="parquet", filesystem=fs) - -# PyArrow native approach -s3 = pafs.S3FileSystem(...) -pads.write_dataset(table, path, format="parquet", filesystem=s3) -``` - -### Multiple Read Methods Tested - -The test attempts reads using multiple PyArrow methods: -- `pq.read_table()` - Direct table reading -- `pq.ParquetDataset()` - Dataset-based reading -- `pads.dataset()` - PyArrow dataset API - -All methods successfully read files written by either filesystem implementation. - -## Practical Implications - -### For Users - -1. **Flexibility**: Users can choose either s3fs or PyArrow native S3 based on their preferences: - - **s3fs**: More mature, widely used, familiar API - - **PyArrow native**: Pure PyArrow solution, fewer dependencies - -2. **Interoperability**: Teams using different tools can seamlessly share Parquet datasets stored in SeaweedFS - -3. **Migration**: Easy to migrate between filesystem implementations without data conversion - -### For SeaweedFS - -1. **S3 Compatibility**: Confirms SeaweedFS's S3 implementation is compatible with major Python data science tools - -2. **Implicit Directory Handling**: The implicit directory fix works correctly for both filesystem implementations - -3. **Standard Compliance**: SeaweedFS handles S3 operations in a way that's compatible with AWS S3 behavior - -## Running the Tests - -### Quick Test (Recommended for Development) - -```bash -cd test/s3/parquet -TEST_QUICK=1 make test-cross-fs-with-server -``` - -### Full Test (All File Sizes) - -```bash -cd test/s3/parquet -make test-cross-fs-with-server -``` - -### Manual Test (Assuming Server is Running) - -```bash -cd test/s3/parquet -make setup-python -make start-seaweedfs-ci - -# In another terminal -TEST_QUICK=1 make test-cross-fs - -# Cleanup -make stop-seaweedfs-safe -``` - -## Environment Variables - -The test supports customization through environment variables: - -- `S3_ENDPOINT_URL`: S3 endpoint (default: `http://localhost:8333`) -- `S3_ACCESS_KEY`: Access key (default: `some_access_key1`) -- `S3_SECRET_KEY`: Secret key (default: `some_secret_key1`) -- `BUCKET_NAME`: Bucket name (default: `test-parquet-bucket`) -- `TEST_QUICK`: Run only small tests (default: `0`, set to `1` for quick mode) - -## Conclusion - -The cross-filesystem compatibility tests demonstrate that **Parquet files written via s3fs and PyArrow native S3 filesystem are completely interchangeable**. This validates that: - -1. The Parquet file format is implementation-agnostic -2. SeaweedFS's S3 API correctly handles both filesystem backends -3. Users have full flexibility in choosing their preferred filesystem implementation - -This compatibility is a testament to: -- PyArrow's consistent file format generation -- SeaweedFS's robust S3 API implementation -- Proper handling of S3 semantics (especially implicit directories) - ---- - -**Test Implementation**: `test_cross_filesystem_compatibility.py` -**Last Updated**: November 21, 2024 -**Status**: ✅ All critical tests passing - diff --git a/test/s3/parquet/FINAL_ROOT_CAUSE_ANALYSIS.md b/test/s3/parquet/FINAL_ROOT_CAUSE_ANALYSIS.md deleted file mode 100644 index 3dff9cb03..000000000 --- a/test/s3/parquet/FINAL_ROOT_CAUSE_ANALYSIS.md +++ /dev/null @@ -1,58 +0,0 @@ -# Final Root Cause Analysis - -## Overview - -This document provides a deep technical analysis of the s3fs compatibility issue with PyArrow Parquet datasets on SeaweedFS, and the solution implemented to resolve it. - -## Root Cause - -When PyArrow writes datasets using `write_dataset()`, it creates implicit directory structures by writing files without explicit directory markers. However, some S3 workflows may create 0-byte directory markers. - -### The Problem - -1. **PyArrow writes dataset files** without creating explicit directory objects -2. **s3fs calls HEAD** on the directory path to check if it exists -3. **If HEAD returns 200** with `Content-Length: 0`, s3fs interprets it as a file (not a directory) -4. **PyArrow fails** when trying to read, reporting "Parquet file size is 0 bytes" - -### AWS S3 Behavior - -AWS S3 returns **404 Not Found** for implicit directories (directories that only exist because they have children but no explicit marker object). This allows s3fs to fall back to LIST operations to detect the directory. - -## The Solution - -### Implementation - -Modified the S3 API HEAD handler in `weed/s3api/s3api_object_handlers.go` to: - -1. **Check if object ends with `/`**: Explicit directory markers return 200 as before -2. **Check if object has children**: If a 0-byte object has children in the filer, treat it as an implicit directory -3. **Return 404 for implicit directories**: This matches AWS S3 behavior and triggers s3fs's LIST fallback - -### Code Changes - -The fix is implemented in the `HeadObjectHandler` function with logic to: -- Detect implicit directories by checking for child entries -- Return 404 (NoSuchKey) for implicit directories -- Preserve existing behavior for explicit directory markers and regular files - -## Performance Considerations - -### Optimization: Child Check Cache -- Child existence checks are performed via filer LIST operations -- Results could be cached for frequently accessed paths -- Trade-off between consistency and performance - -### Impact -- Minimal performance impact for normal file operations -- Slight overhead for HEAD requests on implicit directories (one additional LIST call) -- Overall improvement in PyArrow compatibility outweighs minor performance cost - -## TODO - -- [ ] Add detailed benchmarking results comparing before/after fix -- [ ] Document edge cases discovered during implementation -- [ ] Add architectural diagrams showing the request flow -- [ ] Document alternative solutions considered and why they were rejected -- [ ] Add performance profiling data for child existence checks - diff --git a/test/s3/parquet/MINIO_DIRECTORY_HANDLING.md b/test/s3/parquet/MINIO_DIRECTORY_HANDLING.md deleted file mode 100644 index 04d80cfcb..000000000 --- a/test/s3/parquet/MINIO_DIRECTORY_HANDLING.md +++ /dev/null @@ -1,70 +0,0 @@ -# MinIO Directory Handling Comparison - -## Overview - -This document compares how MinIO handles directory markers versus SeaweedFS's implementation, and explains the different approaches to S3 directory semantics. - -## MinIO's Approach - -MinIO handles implicit directories similarly to AWS S3: - -1. **No explicit directory objects**: Directories are implicit, defined only by object key prefixes -2. **HEAD on directory returns 404**: Consistent with AWS S3 behavior -3. **LIST operations reveal directories**: Directories are discovered through delimiter-based LIST operations -4. **Automatic prefix handling**: MinIO automatically recognizes prefixes as directories - -### MinIO Implementation Details - -- Uses in-memory metadata for fast prefix lookups -- Optimized for LIST operations with common delimiter (`/`) -- No persistent directory objects in storage layer -- Directories "exist" as long as they contain objects - -## SeaweedFS Approach - -SeaweedFS uses a filer-based approach with real directory entries: - -### Before the Fix - -1. **Explicit directory objects**: Could create 0-byte objects as directory markers -2. **HEAD returns 200**: Even for implicit directories -3. **Caused s3fs issues**: s3fs interpreted 0-byte HEAD responses as empty files - -### After the Fix - -1. **Hybrid approach**: Supports both explicit markers (with `/` suffix) and implicit directories -2. **HEAD returns 404 for implicit directories**: Matches AWS S3 and MinIO behavior -3. **Filer integration**: Uses filer's directory metadata to detect implicit directories -4. **s3fs compatibility**: Triggers proper LIST fallback behavior - -## Key Differences - -| Aspect | MinIO | SeaweedFS (After Fix) | -|--------|-------|----------------------| -| Directory Storage | No persistent objects | Filer directory entries | -| Implicit Directory HEAD | 404 Not Found | 404 Not Found | -| Explicit Marker HEAD | Not applicable | 200 OK (with `/` suffix) | -| Child Detection | Prefix scan | Filer LIST operation | -| Performance | In-memory lookups | Filer gRPC calls | - -## Implementation Considerations - -### Advantages of SeaweedFS Approach -- Integrates with existing filer metadata -- Supports both implicit and explicit directories -- Preserves directory metadata and attributes -- Compatible with POSIX filer semantics - -### Trade-offs -- Additional filer communication overhead for HEAD requests -- Complexity of supporting both directory paradigms -- Performance depends on filer efficiency - -## TODO - -- [ ] Add performance benchmark comparison: MinIO vs SeaweedFS -- [ ] Document edge cases where behaviors differ -- [ ] Add example request/response traces for both systems -- [ ] Document migration path for users moving from MinIO to SeaweedFS -- [ ] Add compatibility matrix for different S3 clients - diff --git a/test/s3/parquet/Makefile b/test/s3/parquet/Makefile index 0aa6c8117..708f4aa5c 100644 --- a/test/s3/parquet/Makefile +++ b/test/s3/parquet/Makefile @@ -4,14 +4,9 @@ # Default values SEAWEEDFS_BINARY ?= weed S3_PORT ?= 8333 -FILER_PORT ?= 8888 -VOLUME_PORT ?= 8080 -MASTER_PORT ?= 9333 TEST_TIMEOUT ?= 15m ACCESS_KEY ?= some_access_key1 SECRET_KEY ?= some_secret_key1 -VOLUME_MAX_SIZE_MB ?= 50 -VOLUME_MAX_COUNT ?= 100 BUCKET_NAME ?= test-parquet-bucket ENABLE_SSE_S3 ?= false @@ -68,11 +63,7 @@ help: @echo "Configuration:" @echo " SEAWEEDFS_BINARY=$(SEAWEEDFS_BINARY)" @echo " S3_PORT=$(S3_PORT)" - @echo " FILER_PORT=$(FILER_PORT)" - @echo " VOLUME_PORT=$(VOLUME_PORT)" - @echo " MASTER_PORT=$(MASTER_PORT)" @echo " BUCKET_NAME=$(BUCKET_NAME)" - @echo " VOLUME_MAX_SIZE_MB=$(VOLUME_MAX_SIZE_MB)" @echo " ENABLE_SSE_S3=$(ENABLE_SSE_S3)" @echo " PYTHON=$(PYTHON)" @@ -106,39 +97,25 @@ setup-python: check-python start-seaweedfs-ci: check-binary @echo "$(YELLOW)Starting SeaweedFS server for Parquet testing...$(NC)" - # Clean up any existing processes first (CI-safe) - @echo "Cleaning up any existing processes..." + # Clean up any existing processes first (CI-safe) - aggressive cleanup + @echo "Aggressively cleaning up any existing processes on S3 port $(S3_PORT) and master port 9333..." @if command -v lsof >/dev/null 2>&1; then \ - lsof -ti :$(MASTER_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$(VOLUME_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$(FILER_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$$(( $(MASTER_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$$(( $(VOLUME_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$$(( $(FILER_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ + for attempt in 1 2 3; do \ + lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -9 $$pid 2>/dev/null || true; done; \ + lsof -ti :9333 2>/dev/null | head -5 | while read pid; do kill -9 $$pid 2>/dev/null || true; done; \ + sleep 1; \ + done; \ fi - @sleep 2 + @sleep 3 # Create necessary directories - @mkdir -p /tmp/seaweedfs-test-parquet-master - @mkdir -p /tmp/seaweedfs-test-parquet-volume - @mkdir -p /tmp/seaweedfs-test-parquet-filer + @mkdir -p /tmp/seaweedfs-test-parquet # Clean up any old server logs @rm -f /tmp/seaweedfs-parquet-*.log || true - # Start master server with volume size limit and explicit gRPC port - @echo "Starting master server..." - @nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-parquet-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-parquet-master.log 2>&1 & - @sleep 3 - - # Start volume server with master HTTP port and increased capacity - @echo "Starting volume server..." - @nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-parquet-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 -preStopSeconds=1 > /tmp/seaweedfs-parquet-volume.log 2>&1 & - @sleep 5 - - # Start filer server with embedded S3 - @echo "Starting filer server with embedded S3..." + # Start weed mini with embedded S3 + @echo "Starting weed mini with embedded S3..." @if [ "$(ENABLE_SSE_S3)" = "true" ]; then \ echo " SSE-S3 encryption: ENABLED"; \ printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"buckets":[{"name":"$(BUCKET_NAME)","encryption":{"sseS3":{"enabled":true}}}]}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" > /tmp/seaweedfs-parquet-s3.json; \ @@ -146,96 +123,43 @@ start-seaweedfs-ci: check-binary echo " SSE-S3 encryption: DISABLED"; \ printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}]}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" > /tmp/seaweedfs-parquet-s3.json; \ fi - @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) nohup $(SEAWEEDFS_BINARY) filer -port=$(FILER_PORT) -port.grpc=$$(( $(FILER_PORT) + 10000 )) -master=127.0.0.1:$(MASTER_PORT) -dataCenter=defaultDataCenter -ip=127.0.0.1 -s3 -s3.port=$(S3_PORT) -s3.config=/tmp/seaweedfs-parquet-s3.json > /tmp/seaweedfs-parquet-filer.log 2>&1 & - @sleep 5 - - # Wait for S3 service to be ready - use port-based checking for reliability - @echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)" - @for i in $$(seq 1 20); do \ - if netstat -an 2>/dev/null | grep -q ":$(S3_PORT).*LISTEN" || \ - ss -an 2>/dev/null | grep -q ":$(S3_PORT).*LISTEN" || \ - lsof -i :$(S3_PORT) >/dev/null 2>&1; then \ - echo "$(GREEN)S3 service is listening on port $(S3_PORT)$(NC)"; \ - sleep 1; \ - break; \ + @$(SEAWEEDFS_BINARY) mini \ + -dir=/tmp/seaweedfs-test-parquet \ + -ip.bind=0.0.0.0 \ + -s3.port=$(S3_PORT) \ + -s3.config=/tmp/seaweedfs-parquet-s3.json \ + > /tmp/seaweedfs-parquet-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid + @echo "Waiting for S3 service to be fully ready (max 90 seconds)..." + @bash -c 'for i in $$(seq 1 90); do \ + if curl -s -H "Authorization: AWS4-HMAC-SHA256 Credential=$(ACCESS_KEY)" http://localhost:$(S3_PORT)/ > /dev/null 2>&1; then \ + echo "✅ S3 service is ready"; \ + sleep 2; \ + exit 0; \ fi; \ - if [ $$i -eq 20 ]; then \ - echo "$(RED)S3 service failed to start within 20 seconds$(NC)"; \ - echo "=== Detailed Logs ==="; \ - echo "Master log:"; tail -30 /tmp/seaweedfs-parquet-master.log || true; \ - echo "Volume log:"; tail -30 /tmp/seaweedfs-parquet-volume.log || true; \ - echo "Filer log:"; tail -30 /tmp/seaweedfs-parquet-filer.log || true; \ - echo "=== Port Status ==="; \ - netstat -an 2>/dev/null | grep ":$(S3_PORT)" || \ - ss -an 2>/dev/null | grep ":$(S3_PORT)" || \ - echo "No port listening on $(S3_PORT)"; \ - exit 1; \ - fi; \ - echo "Waiting for S3 service... ($$i/20)"; \ sleep 1; \ - done - - # Additional wait for filer gRPC to be ready - @echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)" - @sleep 2 - - # Wait for volume server to register with master and ensure volume assignment works - @echo "$(YELLOW)Waiting for volume assignment to be ready...$(NC)" - @for i in $$(seq 1 30); do \ - ASSIGN_RESULT=$$(curl -s "http://localhost:$(MASTER_PORT)/dir/assign?count=1" 2>/dev/null); \ - if echo "$$ASSIGN_RESULT" | grep -q '"fid"'; then \ - echo "$(GREEN)Volume assignment is ready$(NC)"; \ - break; \ - fi; \ - if [ $$i -eq 30 ]; then \ - echo "$(RED)Volume assignment not ready after 30 seconds$(NC)"; \ - echo "=== Last assign attempt ==="; \ - echo "$$ASSIGN_RESULT"; \ - echo "=== Master Status ==="; \ - curl -s "http://localhost:$(MASTER_PORT)/dir/status" 2>/dev/null || echo "Failed to get master status"; \ - echo "=== Master Logs ==="; \ - tail -50 /tmp/seaweedfs-parquet-master.log 2>/dev/null || echo "No master log"; \ - echo "=== Volume Logs ==="; \ - tail -50 /tmp/seaweedfs-parquet-volume.log 2>/dev/null || echo "No volume log"; \ - exit 1; \ - fi; \ - echo "Waiting for volume assignment... ($$i/30)"; \ - sleep 1; \ - done - - @echo "$(GREEN)SeaweedFS server started successfully for Parquet testing$(NC)" - @echo "Master: http://localhost:$(MASTER_PORT)" - @echo "Volume: http://localhost:$(VOLUME_PORT)" - @echo "Filer: http://localhost:$(FILER_PORT)" - @echo "S3: http://localhost:$(S3_PORT)" - @echo "Volume Max Size: $(VOLUME_MAX_SIZE_MB)MB" + done; \ + echo "ERROR S3 service failed to start within 90 seconds"; \ + echo "=== Server log output ==="; \ + cat /tmp/seaweedfs-parquet-mini.log 2>/dev/null || echo "No startup log available"; \ + exit 1' start-seaweedfs: check-binary @echo "$(YELLOW)Starting SeaweedFS server for Parquet testing...$(NC)" @# Use port-based cleanup for consistency and safety @echo "Cleaning up any existing processes..." - @lsof -ti :$(MASTER_PORT) 2>/dev/null | xargs -r kill -TERM || true - @lsof -ti :$(VOLUME_PORT) 2>/dev/null | xargs -r kill -TERM || true - @lsof -ti :$(FILER_PORT) 2>/dev/null | xargs -r kill -TERM || true @lsof -ti :$(S3_PORT) 2>/dev/null | xargs -r kill -TERM || true - @# Clean up gRPC ports (HTTP port + 10000) - @lsof -ti :$$(( $(MASTER_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true - @lsof -ti :$$(( $(VOLUME_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true - @lsof -ti :$$(( $(FILER_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true @sleep 2 @$(MAKE) start-seaweedfs-ci stop-seaweedfs: @echo "$(YELLOW)Stopping SeaweedFS server...$(NC)" @# Use port-based cleanup for consistency and safety - @lsof -ti :$(MASTER_PORT) 2>/dev/null | xargs -r kill -TERM || true - @lsof -ti :$(VOLUME_PORT) 2>/dev/null | xargs -r kill -TERM || true - @lsof -ti :$(FILER_PORT) 2>/dev/null | xargs -r kill -TERM || true + @if [ -f /tmp/weed-mini.pid ]; then \ + echo "Stopping weed mini..."; \ + kill $$(cat /tmp/weed-mini.pid) || true; \ + rm -f /tmp/weed-mini.pid; \ + fi @lsof -ti :$(S3_PORT) 2>/dev/null | xargs -r kill -TERM || true - @# Clean up gRPC ports (HTTP port + 10000) - @lsof -ti :$$(( $(MASTER_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true - @lsof -ti :$$(( $(VOLUME_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true - @lsof -ti :$$(( $(FILER_PORT) + 10000 )) 2>/dev/null | xargs -r kill -TERM || true @sleep 2 @echo "$(GREEN)SeaweedFS server stopped$(NC)" @@ -245,22 +169,10 @@ stop-seaweedfs-safe: @# Use port-based cleanup which is safer in CI @if command -v lsof >/dev/null 2>&1; then \ echo "Using lsof for port-based cleanup..."; \ - lsof -ti :$(MASTER_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$(VOLUME_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$(FILER_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ lsof -ti :$(S3_PORT) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$$(( $(MASTER_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$$(( $(VOLUME_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ - lsof -ti :$$(( $(FILER_PORT) + 10000 )) 2>/dev/null | head -5 | while read pid; do kill -TERM $$pid 2>/dev/null || true; done; \ else \ echo "lsof not available, using netstat approach..."; \ - netstat -tlnp 2>/dev/null | grep :$(MASTER_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \ - netstat -tlnp 2>/dev/null | grep :$(VOLUME_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \ - netstat -tlnp 2>/dev/null | grep :$(FILER_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \ netstat -tlnp 2>/dev/null | grep :$(S3_PORT) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \ - netstat -tlnp 2>/dev/null | grep :$$(( $(MASTER_PORT) + 10000 )) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \ - netstat -tlnp 2>/dev/null | grep :$$(( $(VOLUME_PORT) + 10000 )) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \ - netstat -tlnp 2>/dev/null | grep :$$(( $(FILER_PORT) + 10000 )) | awk '{print $$7}' | cut -d/ -f1 | head -5 | while read pid; do [ "$$pid" != "-" ] && kill -TERM $$pid 2>/dev/null || true; done; \ fi @sleep 2 @echo "$(GREEN)SeaweedFS server safely stopped$(NC)" @@ -351,18 +263,14 @@ test-implicit-dir-with-server: build-weed setup-python # Debug targets debug-logs: - @echo "$(YELLOW)=== Master Log ===$(NC)" - @tail -n 50 /tmp/seaweedfs-parquet-master.log || echo "No master log found" - @echo "$(YELLOW)=== Volume Log ===$(NC)" - @tail -n 50 /tmp/seaweedfs-parquet-volume.log || echo "No volume log found" - @echo "$(YELLOW)=== Filer Log ===$(NC)" - @tail -n 50 /tmp/seaweedfs-parquet-filer.log || echo "No filer log found" + @echo "$(YELLOW)=== Mini Log ===$(NC)" + @tail -n 50 /tmp/seaweedfs-parquet-mini.log || echo "No mini log found" debug-status: @echo "$(YELLOW)=== Process Status ===$(NC)" @ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found" @echo "$(YELLOW)=== Port Status ===$(NC)" - @netstat -an | grep -E "($(MASTER_PORT)|$(VOLUME_PORT)|$(FILER_PORT)|$(S3_PORT))" || echo "No ports in use" + @netstat -an | grep -E "($(S3_PORT))" || echo "No ports in use" # Manual test targets for development manual-start: start-seaweedfs diff --git a/test/s3/parquet/TEST_COVERAGE.md b/test/s3/parquet/TEST_COVERAGE.md deleted file mode 100644 index f08a93ab9..000000000 --- a/test/s3/parquet/TEST_COVERAGE.md +++ /dev/null @@ -1,46 +0,0 @@ -# Test Coverage Documentation - -## Overview - -This document provides comprehensive test coverage documentation for the SeaweedFS S3 Parquet integration tests. - -## Test Categories - -### Unit Tests (Go) -- 17 test cases covering S3 API handlers -- Tests for implicit directory handling -- HEAD request behavior validation -- Located in: `weed/s3api/s3api_implicit_directory_test.go` - -### Integration Tests (Python) -- 6 test cases for implicit directory fix -- Tests HEAD request behavior on directory markers -- s3fs directory detection validation -- PyArrow dataset read compatibility -- Located in: `test_implicit_directory_fix.py` - -### End-to-End Tests (Python) -- 20 test cases combining write and read methods -- Small file tests (5 rows): 10 test combinations -- Large file tests (200,000 rows): 10 test combinations -- Tests multiple write methods: `pads.write_dataset`, `pq.write_table+s3fs` -- Tests multiple read methods: `pads.dataset`, `pq.ParquetDataset`, `pq.read_table`, `s3fs+direct`, `s3fs+buffered` -- Located in: `s3_parquet_test.py` - -## Coverage Summary - -| Test Type | Count | Status | -|-----------|-------|--------| -| Unit Tests (Go) | 17 | ✅ Pass | -| Integration Tests (Python) | 6 | ✅ Pass | -| End-to-End Tests (Python) | 20 | ✅ Pass | -| **Total** | **43** | **✅ All Pass** | - -## TODO - -- [ ] Add detailed test execution time metrics -- [ ] Document test data generation strategies -- [ ] Add code coverage percentages for Go tests -- [ ] Document edge cases and corner cases tested -- [ ] Add performance benchmarking results - diff --git a/test/s3/parquet/debug_write_dataset.py b/test/s3/parquet/debug_write_dataset.py new file mode 100644 index 000000000..41762497b --- /dev/null +++ b/test/s3/parquet/debug_write_dataset.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Debug script to understand what pads.write_dataset creates.""" + +import sys +import pyarrow as pa +import pyarrow.dataset as pads +import s3fs + +# Create a simple test table +table = pa.table({'id': [1, 2, 3], 'value': [1.0, 2.0, 3.0]}) + +# Initialize S3 filesystem +fs = s3fs.S3FileSystem( + client_kwargs={'endpoint_url': 'http://localhost:8333'}, + key='some_access_key1', + secret='some_secret_key1', + use_listings_cache=False, +) + +# Create bucket +if not fs.exists('test-bucket'): + fs.mkdir('test-bucket') + +# Write with pads.write_dataset +test_path = 's3://test-bucket/test-write-simple/' +print(f"Writing to: {test_path}") +print(f"Table schema: {table.schema}") +print(f"Table rows: {table.num_rows}") + +try: + pads.write_dataset(table, test_path, format='parquet', filesystem=fs) + print("\n✓ Write succeeded") + + # List all files recursively + print(f"\nListing all files recursively under {test_path}:") + import os + base_path = 'test-bucket/test-write-simple' + def list_recursive(path, indent=0): + try: + items = fs.ls(path, detail=False) + for item in items: + is_dir = fs.isdir(item) + item_name = item.split('/')[-1] if '/' in item else item + if is_dir: + print(f"{' ' * indent}📁 {item_name}/") + list_recursive(item, indent + 1) + else: + # Get file size + try: + info = fs.info(item) + size = info.get('size', 0) + print(f"{' ' * indent}📄 {item_name} ({size} bytes)") + except: + print(f"{' ' * indent}📄 {item_name}") + except Exception as e: + print(f"{' ' * indent}Error listing {path}: {e}") + + list_recursive(base_path) + + # Try to read back with different methods + print(f"\n\nTrying to read back using different methods:") + + # Method 1: pads.dataset with the same path + print(f"\n1. pads.dataset('{test_path}'):") + try: + dataset = pads.dataset(test_path, format='parquet', filesystem=fs) + result = dataset.to_table() + print(f" ✓ Success: {result.num_rows} rows") + except Exception as e: + print(f" ✗ Failed: {e}") + + # Method 2: pads.dataset with the dir containing parquet files + print(f"\n2. pads.dataset without trailing slash:") + test_path_no_slash = 's3://test-bucket/test-write-simple' + try: + dataset = pads.dataset(test_path_no_slash, format='parquet', filesystem=fs) + result = dataset.to_table() + print(f" ✓ Success: {result.num_rows} rows") + except Exception as e: + print(f" ✗ Failed: {e}") + +except Exception as e: + import traceback + print(f"✗ Error: {e}") + traceback.print_exc() + sys.exit(1) diff --git a/test/s3/parquet/test_implicit_directory_fix.py b/test/s3/parquet/test_implicit_directory_fix.py index 9ac8f0346..58f3f2170 100755 --- a/test/s3/parquet/test_implicit_directory_fix.py +++ b/test/s3/parquet/test_implicit_directory_fix.py @@ -60,6 +60,7 @@ def setup_s3(): endpoint_url=S3_ENDPOINT_URL, aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY, + region_name='us-east-1', use_ssl=False ) @@ -182,7 +183,7 @@ def test_explicit_directory_marker(fs, s3_client): logger.info("="*80) # Create an explicit directory marker - logger.info(f"\nCreating explicit directory: {BUCKET_NAME}/explicit_dir/") + logger.info(f"Creating explicit directory: {BUCKET_NAME}/explicit_dir/") try: s3_client.put_object( Bucket=BUCKET_NAME, diff --git a/test/s3/remote_cache/Makefile b/test/s3/remote_cache/Makefile index 1b7a64539..0292c0d35 100644 --- a/test/s3/remote_cache/Makefile +++ b/test/s3/remote_cache/Makefile @@ -10,19 +10,17 @@ all: test-with-server # Configuration WEED_BINARY := ../../../weed/weed_binary +ACCESS_KEY ?= some_access_key1 +SECRET_KEY ?= some_secret_key1 + # Primary SeaweedFS (the one being tested - has remote caching) PRIMARY_S3_PORT := 8333 -PRIMARY_FILER_PORT := 8888 PRIMARY_MASTER_PORT := 9333 -PRIMARY_VOLUME_PORT := 8080 PRIMARY_METRICS_PORT := 9324 PRIMARY_DIR := ./test-primary-data # Secondary SeaweedFS (acts as "remote" S3 storage) REMOTE_S3_PORT := 8334 -REMOTE_FILER_PORT := 8889 -REMOTE_MASTER_PORT := 9334 -REMOTE_VOLUME_PORT := 8081 REMOTE_METRICS_PORT := 9325 REMOTE_DIR := ./test-remote-data @@ -73,18 +71,11 @@ start-remote: check-deps @echo "Starting remote SeaweedFS (secondary instance)..." @rm -f remote-server.pid @mkdir -p $(REMOTE_DIR) - @$(WEED_BINARY) server \ - -s3 \ + @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \ -s3.port=$(REMOTE_S3_PORT) \ -s3.allowDeleteBucketNotEmpty=true \ - -filer \ - -filer.port=$(REMOTE_FILER_PORT) \ - -master.port=$(REMOTE_MASTER_PORT) \ - -volume.port=$(REMOTE_VOLUME_PORT) \ - -master.volumeSizeLimitMB=50 \ - -volume.max=100 \ -dir=$(REMOTE_DIR) \ - -volume.preStopSeconds=1 \ + -ip.bind=0.0.0.0 \ -metricsPort=$(REMOTE_METRICS_PORT) \ > remote-weed.log 2>&1 & echo $$! > remote-server.pid @echo "Waiting for remote SeaweedFS to start..." @@ -93,7 +84,7 @@ start-remote: check-deps echo "Remote SeaweedFS started on port $(REMOTE_S3_PORT)"; \ exit 0; \ fi; \ - sleep 1; \ + sleep 3; \ done; \ echo "ERROR: Remote SeaweedFS failed to start"; \ cat remote-weed.log; \ @@ -114,18 +105,11 @@ start-primary: check-deps @echo "Starting primary SeaweedFS..." @rm -f primary-server.pid @mkdir -p $(PRIMARY_DIR) - @$(WEED_BINARY) server \ - -s3 \ + @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \ -s3.port=$(PRIMARY_S3_PORT) \ -s3.allowDeleteBucketNotEmpty=true \ - -filer \ - -filer.port=$(PRIMARY_FILER_PORT) \ - -master.port=$(PRIMARY_MASTER_PORT) \ - -volume.port=$(PRIMARY_VOLUME_PORT) \ - -master.volumeSizeLimitMB=50 \ - -volume.max=100 \ -dir=$(PRIMARY_DIR) \ - -volume.preStopSeconds=1 \ + -ip.bind=0.0.0.0 \ -metricsPort=$(PRIMARY_METRICS_PORT) \ > primary-weed.log 2>&1 & echo $$! > primary-server.pid @echo "Waiting for primary SeaweedFS to start..." @@ -134,7 +118,7 @@ start-primary: check-deps echo "Primary SeaweedFS started on port $(PRIMARY_S3_PORT)"; \ exit 0; \ fi; \ - sleep 1; \ + sleep 3; \ done; \ echo "ERROR: Primary SeaweedFS failed to start"; \ cat primary-weed.log; \ @@ -156,7 +140,7 @@ setup-remote: @curl -s -X PUT "http://localhost:$(REMOTE_S3_PORT)/$(REMOTE_BUCKET)" || echo "Bucket may already exist" @sleep 1 @echo "Configuring remote storage on primary..." - @printf 'remote.configure -name=seaweedremote -type=s3 -s3.access_key=any -s3.secret_key=any -s3.endpoint=http://localhost:$(REMOTE_S3_PORT) -s3.region=us-east-1\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.configure done" + @printf 'remote.configure -name=seaweedremote -type=s3 -s3.access_key=$(ACCESS_KEY) -s3.secret_key=$(SECRET_KEY) -s3.endpoint=http://localhost:$(REMOTE_S3_PORT) -s3.region=us-east-1\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.configure done" @sleep 2 @echo "Mounting remote bucket on primary..." @printf 'remote.mount -dir=/buckets/remotemounted -remote=seaweedremote/$(REMOTE_BUCKET) -nonempty\nexit\n' | $(WEED_BINARY) shell -master=localhost:$(PRIMARY_MASTER_PORT) 2>&1 || echo "remote.mount done" diff --git a/test/s3/remote_cache/remote_cache_test.go b/test/s3/remote_cache/remote_cache_test.go index 08eca1802..290151ba8 100644 --- a/test/s3/remote_cache/remote_cache_test.go +++ b/test/s3/remote_cache/remote_cache_test.go @@ -34,8 +34,8 @@ const ( remoteEndpoint = "http://localhost:8334" // Credentials (anonymous access for testing) - accessKey = "any" - secretKey = "any" + accessKey = "some_access_key1" + secretKey = "some_secret_key1" // Bucket name - mounted on primary as remote storage testBucket = "remotemounted" @@ -121,17 +121,6 @@ func getFromPrimary(t *testing.T, key string) []byte { return data } -// syncToRemote syncs local data to remote storage -func syncToRemote(t *testing.T) { - t.Log("Syncing to remote storage...") - output, err := runWeedShell(t, "remote.cache.uncache -dir=/buckets/"+testBucket+" -include=*") - if err != nil { - t.Logf("syncToRemote warning: %v", err) - } - t.Log(output) - time.Sleep(1 * time.Second) -} - // uncacheLocal purges the local cache, forcing data to be fetched from remote func uncacheLocal(t *testing.T, pattern string) { t.Logf("Purging local cache for pattern: %s", pattern) diff --git a/test/s3/retention/Makefile b/test/s3/retention/Makefile index 3277e1db0..9854e0aef 100644 --- a/test/s3/retention/Makefile +++ b/test/s3/retention/Makefile @@ -6,11 +6,14 @@ # Configuration WEED_BINARY := ../../../weed/weed_binary S3_PORT := 8333 +ACCESS_KEY ?= some_access_key1 +SECRET_KEY ?= some_secret_key1 MASTER_PORT := 9333 VOLUME_PORT := 8080 FILER_PORT := 8888 TEST_TIMEOUT := 15m TEST_PATTERN := TestRetention +SERVER_DIR := ./test-volume-data/server-data # Default target help: @@ -80,23 +83,16 @@ start-server: check-deps @ls -la ../../../docker/compose/s3.json || echo "⚠️ Config file not found, continuing without it" @echo "🔍 DEBUG: Creating volume directory..." @mkdir -p ./test-volume-data - @echo "🔍 DEBUG: Launching SeaweedFS server in background..." - @echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324" - @$(WEED_BINARY) server \ - -debug \ - -s3 \ + @echo "🔍 DEBUG: Creating server data directory..." + @mkdir -p $(SERVER_DIR) + @echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..." + @echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT)" + @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \ + -dir=$(SERVER_DIR) \ -s3.port=$(S3_PORT) \ - -s3.allowDeleteBucketNotEmpty=true \ - -s3.config=../../../docker/compose/s3.json \ - -filer \ - -filer.maxMB=64 \ - -master.volumeSizeLimitMB=50 \ - -volume.max=100 \ - -dir=./test-volume-data \ - -volume.preStopSeconds=1 \ - -metricsPort=9324 \ - > weed-test.log 2>&1 & echo $$! > weed-server.pid - @echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')" + > weed-test.log 2>&1 & \ + echo $$! > weed-server.pid + @echo "🔍 DEBUG: Server PID: $$(cat weed-test.pid 2>/dev/null || echo 'PID file not found')" @echo "🔍 DEBUG: Checking if PID is still running..." @sleep 2 @if [ -f weed-server.pid ]; then \ @@ -105,7 +101,6 @@ start-server: check-deps else \ echo "⚠️ PID file not found"; \ fi - @echo "🔍 DEBUG: Waiting for server to start (up to 90 seconds)..." @for i in $$(seq 1 90); do \ echo "🔍 DEBUG: Attempt $$i/90 - checking port $(S3_PORT)"; \ if curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1; then \ @@ -123,8 +118,6 @@ start-server: check-deps if [ $$i -eq 15 ]; then \ echo "🔍 DEBUG: After 15 seconds, checking port bindings..."; \ netstat -tlnp 2>/dev/null | grep $(S3_PORT) || echo "Port $(S3_PORT) not bound"; \ - netstat -tlnp 2>/dev/null | grep 9333 || echo "Port 9333 not bound"; \ - netstat -tlnp 2>/dev/null | grep 8080 || echo "Port 8080 not bound"; \ fi; \ if [ $$i -eq 30 ]; then \ echo "⚠️ Server taking longer than expected (30s), checking logs..."; \ diff --git a/test/s3/sse/Makefile b/test/s3/sse/Makefile index 8d0869a82..87c171486 100644 --- a/test/s3/sse/Makefile +++ b/test/s3/sse/Makefile @@ -93,54 +93,35 @@ start-seaweedfs: check-binary @sleep 2 # Create necessary directories - @mkdir -p /tmp/seaweedfs-test-sse-master - @mkdir -p /tmp/seaweedfs-test-sse-volume - @mkdir -p /tmp/seaweedfs-test-sse-filer - - # Start master server with volume size limit and explicit gRPC port - @nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-sse-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-sse-master.log 2>&1 & - @sleep 3 - - # Start volume server with master HTTP port and increased capacity - @nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-sse-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 > /tmp/seaweedfs-sse-volume.log 2>&1 & - @sleep 5 - - # Start filer server (using standard SeaweedFS gRPC port convention: HTTP port + 10000) - @nohup $(SEAWEEDFS_BINARY) filer -port=$(FILER_PORT) -port.grpc=$$(( $(FILER_PORT) + 10000 )) -master=127.0.0.1:$(MASTER_PORT) -dataCenter=defaultDataCenter -ip=127.0.0.1 > /tmp/seaweedfs-sse-filer.log 2>&1 & - @sleep 3 + @mkdir -p /tmp/seaweedfs-test-sse # Create S3 configuration with SSE-KMS support @printf '{"identities":[{"name":"%s","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write"]}],"kms":{"type":"%s","configs":{"keyId":"%s","encryptionContext":{},"bucketKey":false}}}' "$(ACCESS_KEY)" "$(ACCESS_KEY)" "$(SECRET_KEY)" "$(KMS_TYPE)" "$(KMS_KEY_ID)" > /tmp/seaweedfs-sse-s3.json - # Start S3 server with KMS configuration - @nohup $(SEAWEEDFS_BINARY) s3 -port=$(S3_PORT) -filer=127.0.0.1:$(FILER_PORT) -config=/tmp/seaweedfs-sse-s3.json -ip.bind=127.0.0.1 > /tmp/seaweedfs-sse-s3.log 2>&1 & - @sleep 5 - - # Wait for S3 service to be ready - @echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)" + # Start weed mini + @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(SEAWEEDFS_BINARY) mini \ + -dir=/tmp/seaweedfs-test-sse \ + -s3.port=$(S3_PORT) \ + -s3.config=/tmp/seaweedfs-sse-s3.json \ + > /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid + + @echo "Checking S3 service is ready..." @for i in $$(seq 1 30); do \ - if curl -s -f http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ - echo "$(GREEN)S3 service is ready$(NC)"; \ + if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ + echo "✅ S3 service is ready"; \ break; \ fi; \ - echo "Waiting for S3 service... ($$i/30)"; \ sleep 1; \ done - - # Additional wait for filer gRPC to be ready - @echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)" - @sleep 2 - @echo "$(GREEN)SeaweedFS server started successfully for SSE testing$(NC)" - @echo "Master: http://localhost:$(MASTER_PORT)" - @echo "Volume: http://localhost:$(VOLUME_PORT)" - @echo "Filer: http://localhost:$(FILER_PORT)" - @echo "S3: http://localhost:$(S3_PORT)" - @echo "Volume Max Size: $(VOLUME_MAX_SIZE_MB)MB" - @echo "SSE-KMS Support: Enabled" stop-seaweedfs: @echo "$(YELLOW)Stopping SeaweedFS server...$(NC)" @# Use port-based cleanup for consistency and safety + @if [ -f /tmp/weed-mini.pid ]; then \ + echo "Stopping weed mini..."; \ + kill $$(cat /tmp/weed-mini.pid) || true; \ + rm -f /tmp/weed-mini.pid; \ + fi @lsof -ti :$(MASTER_PORT) | xargs -r kill -TERM || true @lsof -ti :$(VOLUME_PORT) | xargs -r kill -TERM || true @lsof -ti :$(FILER_PORT) | xargs -r kill -TERM || true @@ -345,71 +326,33 @@ start-seaweedfs-ci: check-binary @echo "$(YELLOW)Starting SeaweedFS server for CI testing...$(NC)" # Create necessary directories - @mkdir -p /tmp/seaweedfs-test-sse-master - @mkdir -p /tmp/seaweedfs-test-sse-volume - @mkdir -p /tmp/seaweedfs-test-sse-filer + @mkdir -p /tmp/seaweedfs-test-sse # Clean up any old server logs @rm -f /tmp/seaweedfs-sse-*.log || true - # Start master server with volume size limit and explicit gRPC port - @echo "Starting master server..." - @nohup $(SEAWEEDFS_BINARY) master -port=$(MASTER_PORT) -port.grpc=$$(( $(MASTER_PORT) + 10000 )) -mdir=/tmp/seaweedfs-test-sse-master -volumeSizeLimitMB=$(VOLUME_MAX_SIZE_MB) -ip=127.0.0.1 -peers=none > /tmp/seaweedfs-sse-master.log 2>&1 & - @sleep 3 - - # Start volume server with master HTTP port and increased capacity - @echo "Starting volume server..." - @nohup $(SEAWEEDFS_BINARY) volume -port=$(VOLUME_PORT) -master=127.0.0.1:$(MASTER_PORT) -dir=/tmp/seaweedfs-test-sse-volume -max=$(VOLUME_MAX_COUNT) -ip=127.0.0.1 > /tmp/seaweedfs-sse-volume.log 2>&1 & - @sleep 5 - # Create S3 JSON configuration with KMS (Local provider) and basic identity for embedded S3 @sed -e 's/ACCESS_KEY_PLACEHOLDER/$(ACCESS_KEY)/g' \ -e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \ s3-config-template.json > /tmp/seaweedfs-s3.json - # Start filer server with embedded S3 using the JSON config (with verbose logging) - @echo "Starting filer server with embedded S3..." - @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 nohup $(SEAWEEDFS_BINARY) filer -port=$(FILER_PORT) -port.grpc=$$(( $(FILER_PORT) + 10000 )) -master=127.0.0.1:$(MASTER_PORT) -dataCenter=defaultDataCenter -ip=127.0.0.1 -s3 -s3.port=$(S3_PORT) -s3.config=/tmp/seaweedfs-s3.json > /tmp/seaweedfs-sse-filer.log 2>&1 & - @sleep 5 + # Start weed mini with embedded S3 using the JSON config (with verbose logging) + @echo "Starting weed mini with embedded S3..." + @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \ + -dir=/tmp/seaweedfs-test-sse \ + -s3.port=$(S3_PORT) \ + -s3.config=/tmp/seaweedfs-s3.json \ + -ip=127.0.0.1 \ + > /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid - # Wait for S3 service to be ready - use port-based checking for reliability - @echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)" - @for i in $$(seq 1 20); do \ - if netstat -an 2>/dev/null | grep -q ":$(S3_PORT).*LISTEN" || \ - ss -an 2>/dev/null | grep -q ":$(S3_PORT).*LISTEN" || \ - lsof -i :$(S3_PORT) >/dev/null 2>&1; then \ - echo "$(GREEN)S3 service is listening on port $(S3_PORT)$(NC)"; \ - sleep 1; \ + @echo "Checking S3 service is ready..." + @for i in $$(seq 1 30); do \ + if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ + echo "✅ S3 service is ready"; \ break; \ fi; \ - if [ $$i -eq 20 ]; then \ - echo "$(RED)S3 service failed to start within 20 seconds$(NC)"; \ - echo "=== Detailed Logs ==="; \ - echo "Master log:"; tail -30 /tmp/seaweedfs-sse-master.log || true; \ - echo "Volume log:"; tail -30 /tmp/seaweedfs-sse-volume.log || true; \ - echo "Filer log:"; tail -30 /tmp/seaweedfs-sse-filer.log || true; \ - echo "=== Port Status ==="; \ - netstat -an 2>/dev/null | grep ":$(S3_PORT)" || \ - ss -an 2>/dev/null | grep ":$(S3_PORT)" || \ - echo "No port listening on $(S3_PORT)"; \ - echo "=== Process Status ==="; \ - ps aux | grep -E "weed.*(filer|s3).*$(S3_PORT)" | grep -v grep || echo "No S3 process found"; \ - exit 1; \ - fi; \ - echo "Waiting for S3 service... ($$i/20)"; \ sleep 1; \ done - - # Additional wait for filer gRPC to be ready - @echo "$(YELLOW)Waiting for filer gRPC to be ready...$(NC)" - @sleep 2 - @echo "$(GREEN)SeaweedFS server started successfully for SSE testing$(NC)" - @echo "Master: http://localhost:$(MASTER_PORT)" - @echo "Volume: http://localhost:$(VOLUME_PORT)" - @echo "Filer: http://localhost:$(FILER_PORT)" - @echo "S3: http://localhost:$(S3_PORT)" - @echo "Volume Max Size: $(VOLUME_MAX_SIZE_MB)MB" - @echo "SSE-KMS Support: Enabled" # GitHub Actions compatible quick test subset test-quick-with-server: build-weed @@ -527,3 +470,36 @@ dev-kms: setup-openbao @echo "OpenBao: $(OPENBAO_ADDR)" @echo "Token: $(OPENBAO_TOKEN)" @echo "Use 'make test-ssekms-integration' to run tests" + +# Volume encryption integration tests +test-volume-encryption: build-weed + @echo "🚀 Starting S3 volume encryption integration tests..." + @echo "Starting SeaweedFS cluster with volume encryption enabled..." + @# Start server with -s3.encryptVolumeData flag + @mkdir -p /tmp/seaweedfs-test-sse + @rm -f /tmp/seaweedfs-sse-*.log || true + @sed -e 's/ACCESS_KEY_PLACEHOLDER/$(ACCESS_KEY)/g' \ + -e 's/SECRET_KEY_PLACEHOLDER/$(SECRET_KEY)/g' \ + s3-config-template.json > /tmp/seaweedfs-s3.json + @echo "Starting weed mini with S3 volume encryption..." + @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) GLOG_v=4 $(SEAWEEDFS_BINARY) mini \ + -dir=/tmp/seaweedfs-test-sse \ + -s3.port=$(S3_PORT) \ + -s3.config=/tmp/seaweedfs-s3.json \ + -s3.encryptVolumeData \ + -ip=127.0.0.1 \ + > /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid + @echo "Checking S3 service is ready..." + @for i in $$(seq 1 30); do \ + if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ + echo "✅ S3 service is ready"; \ + break; \ + fi; \ + sleep 1; \ + done + @echo "Running volume encryption integration tests..." + @trap '$(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true' EXIT; \ + cd $(SEAWEEDFS_ROOT) && go test -v -tags=integration -timeout=10m -run "TestS3VolumeEncryption" ./test/s3/sse || exit 1; \ + echo "✅ Volume encryption tests completed successfully"; \ + $(MAKE) -C $(TEST_DIR) stop-seaweedfs-safe || true + diff --git a/test/s3/sse/s3_volume_encryption_test.go b/test/s3/sse/s3_volume_encryption_test.go new file mode 100644 index 000000000..349763c10 --- /dev/null +++ b/test/s3/sse/s3_volume_encryption_test.go @@ -0,0 +1,448 @@ +//go:build integration +// +build integration + +package sse + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" +) + +// TestS3VolumeEncryptionRoundtrip tests that data uploaded with encryptVolumeData +// enabled can be read back correctly. This requires the S3 server to be started with +// -encryptVolumeData=true for the test to verify encryption is working. +// +// To run this test: +// 1. Start SeaweedFS: weed server -s3 -s3.encryptVolumeData=true +// 2. Run: go test -v -run TestS3VolumeEncryptionRoundtrip +func TestS3VolumeEncryptionRoundtrip(t *testing.T) { + svc := getS3Client(t) + bucket := fmt.Sprintf("volume-encryption-test-%d", time.Now().Unix()) + + // Create bucket + _, err := svc.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + t.Fatalf("Failed to create bucket: %v", err) + } + defer cleanupBucket(t, svc, bucket) + + testCases := []struct { + name string + key string + content string + rangeReq string // Optional range request + }{ + { + name: "small file", + key: "small.txt", + content: "Hello, encrypted world!", + }, + { + name: "medium file", + key: "medium.txt", + content: strings.Repeat("SeaweedFS volume encryption test content. ", 1000), + }, + { + name: "binary content", + key: "binary.bin", + content: string([]byte{0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD, 0x00, 0x80}), + }, + { + name: "range request", + key: "range-test.txt", + content: "0123456789ABCDEFGHIJ", + rangeReq: "bytes=5-10", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Upload + _, err := svc.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(tc.key), + Body: strings.NewReader(tc.content), + }) + if err != nil { + t.Fatalf("PutObject failed: %v", err) + } + + // Download + getInput := &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(tc.key), + } + if tc.rangeReq != "" { + getInput.Range = aws.String(tc.rangeReq) + } + + result, err := svc.GetObject(getInput) + if err != nil { + t.Fatalf("GetObject failed: %v", err) + } + defer result.Body.Close() + + data, err := io.ReadAll(result.Body) + if err != nil { + t.Fatalf("Failed to read response body: %v", err) + } + + // Verify content + expected := tc.content + if tc.rangeReq != "" { + // For "bytes=5-10", we expect characters at positions 5-10 (inclusive) + expected = tc.content[5:11] + } + + if string(data) != expected { + t.Errorf("Content mismatch:\n expected: %q\n got: %q", expected, string(data)) + } else { + t.Logf("Successfully uploaded and downloaded %s (%d bytes)", tc.key, len(data)) + } + }) + } +} + +// TestS3VolumeEncryptionMultiChunk tests large files that span multiple chunks +// to ensure encryption works correctly across chunk boundaries. +func TestS3VolumeEncryptionMultiChunk(t *testing.T) { + svc := getS3Client(t) + bucket := fmt.Sprintf("volume-encryption-multichunk-%d", time.Now().Unix()) + + // Create bucket + _, err := svc.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + t.Fatalf("Failed to create bucket: %v", err) + } + defer cleanupBucket(t, svc, bucket) + + // Create a file larger than default chunk size (8MB) + // Use 10MB to ensure multiple chunks + largeContent := make([]byte, 10*1024*1024) + for i := range largeContent { + largeContent[i] = byte(i % 256) + } + + key := "large-file.bin" + + // Upload + _, err = svc.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(largeContent), + }) + if err != nil { + t.Fatalf("PutObject failed for large file: %v", err) + } + + // Download full file + result, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + if err != nil { + t.Fatalf("GetObject failed: %v", err) + } + defer result.Body.Close() + + downloadedData, err := io.ReadAll(result.Body) + if err != nil { + t.Fatalf("Failed to read response body: %v", err) + } + + if len(downloadedData) != len(largeContent) { + t.Errorf("Size mismatch: expected %d, got %d", len(largeContent), len(downloadedData)) + } + + if !bytes.Equal(downloadedData, largeContent) { + t.Errorf("Content mismatch in multi-chunk file") + // Find first mismatch + for i := 0; i < len(downloadedData) && i < len(largeContent); i++ { + if downloadedData[i] != largeContent[i] { + t.Errorf("First mismatch at byte %d: expected %02x, got %02x", i, largeContent[i], downloadedData[i]) + break + } + } + } else { + t.Logf("Successfully uploaded and downloaded %d byte multi-chunk file", len(downloadedData)) + } + + // Test range request spanning chunk boundary (around 8MB) + rangeStart := int64(8*1024*1024 - 1000) + rangeEnd := int64(8*1024*1024 + 1000) + rangeResult, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Range: aws.String(fmt.Sprintf("bytes=%d-%d", rangeStart, rangeEnd)), + }) + if err != nil { + t.Fatalf("Range GetObject failed: %v", err) + } + defer rangeResult.Body.Close() + + rangeData, err := io.ReadAll(rangeResult.Body) + if err != nil { + t.Fatalf("Failed to read range response: %v", err) + } + + expectedRange := largeContent[rangeStart : rangeEnd+1] + if !bytes.Equal(rangeData, expectedRange) { + t.Errorf("Range request content mismatch at chunk boundary") + } else { + t.Logf("Successfully retrieved range spanning chunk boundary (%d bytes)", len(rangeData)) + } +} + +// TestS3VolumeEncryptionMultiChunkRangeRead tests range reads that span multiple chunks: +// - Part of chunk 1 +// - Whole chunk 2 +// - Part of chunk 3 +// This is critical for verifying that cipher decryption works correctly when +// reading across chunk boundaries with the cipherKey from each chunk. +func TestS3VolumeEncryptionMultiChunkRangeRead(t *testing.T) { + svc := getS3Client(t) + bucket := fmt.Sprintf("volume-encryption-multirange-%d", time.Now().Unix()) + + // Create bucket + _, err := svc.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + t.Fatalf("Failed to create bucket: %v", err) + } + defer cleanupBucket(t, svc, bucket) + + // Default chunk size is 8MB. Create a file with 3+ chunks (25MB) + // to ensure we have multiple complete chunks + const chunkSize = 8 * 1024 * 1024 // 8MB + const fileSize = 25 * 1024 * 1024 // 25MB = 3 chunks + partial 4th + + largeContent := make([]byte, fileSize) + for i := range largeContent { + // Use recognizable pattern: each byte encodes its position + largeContent[i] = byte(i % 256) + } + + key := "multi-chunk-range-test.bin" + + // Upload + _, err = svc.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(largeContent), + }) + if err != nil { + t.Fatalf("PutObject failed: %v", err) + } + + t.Logf("Uploaded %d byte file (%d chunks of %d bytes)", fileSize, (fileSize+chunkSize-1)/chunkSize, chunkSize) + + // Test cases for range reads spanning multiple chunks + testCases := []struct { + name string + rangeStart int64 + rangeEnd int64 + desc string + }{ + { + name: "part_chunk1_whole_chunk2_part_chunk3", + rangeStart: chunkSize - 1000, // 1000 bytes before end of chunk 1 + rangeEnd: 2*chunkSize + 1000, // 1000 bytes into chunk 3 + desc: "Spans from end of chunk 1 through entire chunk 2 into beginning of chunk 3", + }, + { + name: "last_byte_chunk1_through_first_byte_chunk3", + rangeStart: chunkSize - 1, // Last byte of chunk 1 + rangeEnd: 2 * chunkSize, // First byte of chunk 3 + desc: "Minimal span: last byte of chunk 1, entire chunk 2, first byte of chunk 3", + }, + { + name: "middle_chunk1_to_middle_chunk3", + rangeStart: chunkSize / 2, // Middle of chunk 1 + rangeEnd: 2*chunkSize + chunkSize/2, // Middle of chunk 3 + desc: "From middle of chunk 1 to middle of chunk 3", + }, + { + name: "half_chunk1_whole_chunk2_half_chunk3", + rangeStart: chunkSize / 2, // Half of chunk 1 + rangeEnd: 2*chunkSize + chunkSize/2 - 1, // Half of chunk 3 + desc: "Half of each boundary chunk, whole middle chunk", + }, + { + name: "single_byte_each_boundary", + rangeStart: chunkSize - 1, // 1 byte from chunk 1 + rangeEnd: 2 * chunkSize, // 1 byte from chunk 3 + desc: "1 byte from chunk 1, entire chunk 2, 1 byte from chunk 3", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Logf("Test: %s", tc.desc) + t.Logf("Range: bytes=%d-%d (spanning bytes at offsets across chunk boundaries)", tc.rangeStart, tc.rangeEnd) + + result, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Range: aws.String(fmt.Sprintf("bytes=%d-%d", tc.rangeStart, tc.rangeEnd)), + }) + if err != nil { + t.Fatalf("Range GetObject failed: %v", err) + } + defer result.Body.Close() + + rangeData, err := io.ReadAll(result.Body) + if err != nil { + t.Fatalf("Failed to read range response: %v", err) + } + + expectedLen := tc.rangeEnd - tc.rangeStart + 1 + if int64(len(rangeData)) != expectedLen { + t.Errorf("Size mismatch: expected %d bytes, got %d", expectedLen, len(rangeData)) + } + + expectedRange := largeContent[tc.rangeStart : tc.rangeEnd+1] + if !bytes.Equal(rangeData, expectedRange) { + t.Errorf("Content mismatch in multi-chunk range read") + // Find first mismatch for debugging + for i := 0; i < len(rangeData) && i < len(expectedRange); i++ { + if rangeData[i] != expectedRange[i] { + globalOffset := tc.rangeStart + int64(i) + chunkNum := globalOffset / chunkSize + offsetInChunk := globalOffset % chunkSize + t.Errorf("First mismatch at byte %d (chunk %d, offset %d in chunk): expected %02x, got %02x", + i, chunkNum, offsetInChunk, expectedRange[i], rangeData[i]) + break + } + } + } else { + t.Logf("✓ Successfully read %d bytes spanning chunks (offsets %d-%d)", len(rangeData), tc.rangeStart, tc.rangeEnd) + } + }) + } +} + +// TestS3VolumeEncryptionCopy tests that copying encrypted objects works correctly. +func TestS3VolumeEncryptionCopy(t *testing.T) { + svc := getS3Client(t) + bucket := fmt.Sprintf("volume-encryption-copy-%d", time.Now().Unix()) + + // Create bucket + _, err := svc.CreateBucket(&s3.CreateBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + t.Fatalf("Failed to create bucket: %v", err) + } + defer cleanupBucket(t, svc, bucket) + + srcKey := "source-object.txt" + dstKey := "copied-object.txt" + content := "Content to be copied with volume-level encryption" + + // Upload source + _, err = svc.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(srcKey), + Body: strings.NewReader(content), + }) + if err != nil { + t.Fatalf("PutObject failed: %v", err) + } + + // Copy object + _, err = svc.CopyObject(&s3.CopyObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(dstKey), + CopySource: aws.String(fmt.Sprintf("%s/%s", bucket, srcKey)), + }) + if err != nil { + t.Fatalf("CopyObject failed: %v", err) + } + + // Read copied object + result, err := svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(dstKey), + }) + if err != nil { + t.Fatalf("GetObject failed for copied object: %v", err) + } + defer result.Body.Close() + + data, err := io.ReadAll(result.Body) + if err != nil { + t.Fatalf("Failed to read copied object: %v", err) + } + + if string(data) != content { + t.Errorf("Copied content mismatch:\n expected: %q\n got: %q", content, string(data)) + } else { + t.Logf("Successfully copied encrypted object") + } +} + +// Helper functions + +func getS3Client(t *testing.T) *s3.S3 { + // Use credentials that match the Makefile configuration + // ACCESS_KEY ?= some_access_key1 + // SECRET_KEY ?= some_secret_key1 + sess, err := session.NewSession(&aws.Config{ + Region: aws.String("us-east-1"), + Endpoint: aws.String("http://localhost:8333"), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + Credentials: credentials.NewStaticCredentials("some_access_key1", "some_secret_key1", ""), + }) + if err != nil { + t.Fatalf("Failed to create session: %v", err) + } + return s3.New(sess) +} + +func cleanupBucket(t *testing.T, svc *s3.S3, bucket string) { + ctx := context.Background() + _ = ctx + + // List and delete all objects + listResult, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + }) + if err != nil { + t.Logf("Warning: failed to list objects for cleanup: %v", err) + return + } + + for _, obj := range listResult.Contents { + _, err := svc.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: obj.Key, + }) + if err != nil { + t.Logf("Warning: failed to delete object %s: %v", *obj.Key, err) + } + } + + // Delete bucket + _, err = svc.DeleteBucket(&s3.DeleteBucketInput{ + Bucket: aws.String(bucket), + }) + if err != nil { + t.Logf("Warning: failed to delete bucket %s: %v", bucket, err) + } +} diff --git a/test/s3/tagging/Makefile b/test/s3/tagging/Makefile index c495d1a40..0ae7b1518 100644 --- a/test/s3/tagging/Makefile +++ b/test/s3/tagging/Makefile @@ -72,41 +72,17 @@ start-server: check-deps fi @echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)" @ls -la $(WEED_BINARY) || (echo "❌ Binary not found!" && exit 1) - @echo "🔍 DEBUG: Checking config file at ../../../docker/compose/s3.json" - @ls -la ../../../docker/compose/s3.json || echo "⚠️ Config file not found, continuing without it" @echo "🔍 DEBUG: Creating volume directory..." @mkdir -p ./test-volume-data @echo "🔍 DEBUG: Launching SeaweedFS server in background..." - @echo "🔍 DEBUG: Command: $(WEED_BINARY) server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 -dir=./test-volume-data -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 -volume.max=100 -volume.preStopSeconds=1 -master.port=$(MASTER_PORT) -volume.port=$(VOLUME_PORT) -filer.port=$(FILER_PORT) -s3.port=$(S3_PORT) -metricsPort=9329 -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -master.peers=none" - @$(WEED_BINARY) server \ - -filer \ + @echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=./test-volume-data -s3.port=$(S3_PORT)" + @$(WEED_BINARY) mini \ -filer.maxMB=64 \ - -s3 \ - -ip.bind 0.0.0.0 \ -dir=./test-volume-data \ -master.raftHashicorp \ - -master.electionTimeout 1s \ - -master.volumeSizeLimitMB=100 \ - -volume.max=100 \ - -volume.preStopSeconds=1 \ - -master.port=$(MASTER_PORT) \ - -volume.port=$(VOLUME_PORT) \ - -filer.port=$(FILER_PORT) \ -s3.port=$(S3_PORT) \ - -metricsPort=9329 \ - -s3.allowDeleteBucketNotEmpty=true \ - -s3.config=../../../docker/compose/s3.json \ - -master.peers=none \ - > weed-test.log 2>&1 & echo $$! > weed-server.pid - @echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')" - @echo "🔍 DEBUG: Checking if PID is still running..." - @sleep 2 - @if [ -f weed-server.pid ]; then \ - SERVER_PID=$$(cat weed-server.pid); \ - ps -p $$SERVER_PID || echo "⚠️ Server PID $$SERVER_PID not found after 2 seconds"; \ - else \ - echo "⚠️ PID file not found"; \ - fi + > weed-test.log 2>&1 & \ + echo $$! > weed-server.pid @echo "🔍 DEBUG: Waiting for server to start (up to 90 seconds)..." @for i in $$(seq 1 90); do \ echo "🔍 DEBUG: Attempt $$i/90 - checking port $(S3_PORT)"; \ @@ -125,8 +101,6 @@ start-server: check-deps if [ $$i -eq 15 ]; then \ echo "🔍 DEBUG: After 15 seconds, checking port bindings..."; \ netstat -tlnp 2>/dev/null | grep $(S3_PORT) || echo "Port $(S3_PORT) not bound"; \ - netstat -tlnp 2>/dev/null | grep $(MASTER_PORT) || echo "Port $(MASTER_PORT) not bound"; \ - netstat -tlnp 2>/dev/null | grep $(VOLUME_PORT) || echo "Port $(VOLUME_PORT) not bound"; \ fi; \ if [ $$i -eq 30 ]; then \ echo "⚠️ Server taking longer than expected (30s), checking logs..."; \ @@ -141,7 +115,7 @@ start-server: check-deps echo "🔍 DEBUG: Final process check:"; \ ps aux | grep weed | grep -v grep || echo "No weed processes found"; \ echo "🔍 DEBUG: Final port check:"; \ - netstat -tlnp 2>/dev/null | grep -E "($(S3_PORT)|$(MASTER_PORT)|$(VOLUME_PORT))" || echo "No ports bound"; \ + netstat -tlnp 2>/dev/null | grep -E "($(S3_PORT))" || echo "No ports bound"; \ echo "=== Full server logs ==="; \ if [ -f weed-test.log ]; then \ cat weed-test.log; \ diff --git a/test/s3/tagging/s3_tagging_test.go b/test/s3/tagging/s3_tagging_test.go index c490ca1aa..4606ec800 100644 --- a/test/s3/tagging/s3_tagging_test.go +++ b/test/s3/tagging/s3_tagging_test.go @@ -63,18 +63,12 @@ func getS3Client(t *testing.T) *s3.Client { defaultConfig.SecretKey, "", )), - config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc( - func(service, region string, options ...interface{}) (aws.Endpoint, error) { - return aws.Endpoint{ - URL: defaultConfig.Endpoint, - SigningRegion: defaultConfig.Region, - }, nil - })), ) require.NoError(t, err) client := s3.NewFromConfig(cfg, func(o *s3.Options) { o.UsePathStyle = true + o.BaseEndpoint = aws.String(defaultConfig.Endpoint) }) return client } @@ -113,6 +107,33 @@ func cleanupTestBucket(t *testing.T, client *s3.Client, bucketName string) { } } + // Delete all versions and delete markers if versioning is enabled + listVersionsResp, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{ + Bucket: aws.String(bucketName), + }) + if err == nil { + for _, version := range listVersionsResp.Versions { + _, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: version.Key, + VersionId: version.VersionId, + }) + if err != nil { + t.Logf("Warning: failed to delete version %s: %v", *version.Key, err) + } + } + for _, marker := range listVersionsResp.DeleteMarkers { + _, err := client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String(bucketName), + Key: marker.Key, + VersionId: marker.VersionId, + }) + if err != nil { + t.Logf("Warning: failed to delete marker %s: %v", *marker.Key, err) + } + } + } + // Then delete the bucket _, err = client.DeleteBucket(context.TODO(), &s3.DeleteBucketInput{ Bucket: aws.String(bucketName), diff --git a/test/s3/tagging/s3_tagging_versioning_test.go b/test/s3/tagging/s3_tagging_versioning_test.go new file mode 100644 index 000000000..5aa49f956 --- /dev/null +++ b/test/s3/tagging/s3_tagging_versioning_test.go @@ -0,0 +1,434 @@ +package tagging + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// This is the fix for GitHub issue #7868 where tagging failed with "no entry is found in filer store" +// TestPutObjectTaggingOnVersionedBucket tests setting tags on objects in a versioned bucket +func TestPutObjectTaggingOnVersionedBucket(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + // Put object in versioned bucket + objectKey := "versioned-object-with-tags" + objectContent := "Hello, Versioned World!" + _, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader(objectContent), + Key: aws.String(objectKey), + Bucket: aws.String(bucketName), + }) + require.NoError(t, err, "Should be able to put object in versioned bucket") + + // Set tags on the object in versioned bucket + _, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + Tagging: &types.Tagging{ + TagSet: []types.Tag{ + { + Key: aws.String("env"), + Value: aws.String("production"), + }, + { + Key: aws.String("team"), + Value: aws.String("platform"), + }, + }, + }, + }) + require.NoError(t, err, "Should be able to put tags on versioned object") + + // Get the tags back + tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Should be able to get tags from versioned object") + + // Verify tags + assert.Len(t, tagResp.TagSet, 2, "Should have 2 tags") + tagMap := make(map[string]string) + for _, tag := range tagResp.TagSet { + tagMap[*tag.Key] = *tag.Value + } + assert.Equal(t, "production", tagMap["env"], "env tag should be 'production'") + assert.Equal(t, "platform", tagMap["team"], "team tag should be 'platform'") +} + +// TestPutObjectTaggingOnSpecificVersionInVersionedBucket tests setting tags on a specific version +func TestPutObjectTaggingOnSpecificVersionInVersionedBucket(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + // Create multiple versions of the object + objectKey := "multi-version-object" + version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("Version 1"), + Key: aws.String(objectKey), + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + + // Small delay to ensure different version IDs + time.Sleep(50 * time.Millisecond) + versionId1 := *version1Resp.VersionId + + version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("Version 2"), + Key: aws.String(objectKey), + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + versionId2 := *version2Resp.VersionId + + // Set tags on version 1 + _, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + Tagging: &types.Tagging{ + TagSet: []types.Tag{ + { + Key: aws.String("version"), + Value: aws.String("v1"), + }, + }, + }, + }) + require.NoError(t, err, "Should be able to put tags on specific version 1") + + // Set tags on version 2 + _, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + Tagging: &types.Tagging{ + TagSet: []types.Tag{ + { + Key: aws.String("version"), + Value: aws.String("v2"), + }, + }, + }, + }) + require.NoError(t, err, "Should be able to put tags on specific version 2") + + // Get tags from version 1 + tagResp1, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + }) + require.NoError(t, err, "Should be able to get tags from version 1") + assert.Len(t, tagResp1.TagSet, 1, "Version 1 should have 1 tag") + assert.Equal(t, "v1", *tagResp1.TagSet[0].Value, "Version 1 tag value should be 'v1'") + + // Get tags from version 2 + tagResp2, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + }) + require.NoError(t, err, "Should be able to get tags from version 2") + assert.Len(t, tagResp2.TagSet, 1, "Version 2 should have 1 tag") + assert.Equal(t, "v2", *tagResp2.TagSet[0].Value, "Version 2 tag value should be 'v2'") +} + +// TestDeleteObjectTaggingOnVersionedBucket tests deleting tags from versioned objects +func TestDeleteObjectTaggingOnVersionedBucket(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + // Put object with tags in versioned bucket + objectKey := "versioned-object-tag-delete" + objectContent := "Hello, Delete Tags!" + _, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader(objectContent), + Key: aws.String(objectKey), + Bucket: aws.String(bucketName), + Tagging: aws.String("env=dev&purpose=testing"), + }) + require.NoError(t, err) + + // Verify tags exist + tagResp, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + }) + require.NoError(t, err) + assert.Len(t, tagResp.TagSet, 2, "Should have 2 tags before deletion") + + // Delete tags from the versioned object + _, err = client.DeleteObjectTagging(context.TODO(), &s3.DeleteObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Should be able to delete tags from versioned object") + + // Verify tags are deleted + tagResp, err = client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + }) + require.NoError(t, err) + assert.Len(t, tagResp.TagSet, 0, "Should have 0 tags after deletion") +} + +// TestDeleteObjectTaggingOnSpecificVersionInVersionedBucket tests deleting tags on a specific version +func TestDeleteObjectTaggingOnSpecificVersionInVersionedBucket(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + // Create two versions with tags + objectKey := "versioned-multi-delete-tags" + version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("Version 1"), + Key: aws.String(objectKey), + Bucket: aws.String(bucketName), + Tagging: aws.String("version=v1&keep=true"), + }) + require.NoError(t, err) + + // Small delay to ensure different version IDs + time.Sleep(50 * time.Millisecond) + versionId1 := *version1Resp.VersionId + + version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("Version 2"), + Key: aws.String(objectKey), + Bucket: aws.String(bucketName), + Tagging: aws.String("version=v2&keep=true"), + }) + require.NoError(t, err) + versionId2 := *version2Resp.VersionId + + // Delete tags from version 1 only + _, err = client.DeleteObjectTagging(context.TODO(), &s3.DeleteObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + }) + require.NoError(t, err, "Should be able to delete tags from version 1") + + // Verify version 1 has no tags + tagResp1, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + }) + require.NoError(t, err) + assert.Len(t, tagResp1.TagSet, 0, "Version 1 should have 0 tags after deletion") + + // Verify version 2 still has tags + tagResp2, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + }) + require.NoError(t, err) + assert.Len(t, tagResp2.TagSet, 2, "Version 2 should still have 2 tags") +} + +// TestGetObjectTaggingOnVersionedBucket tests retrieving tags from versioned objects +func TestGetObjectTaggingOnVersionedBucket(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + // Create multiple versions + objectKey := "versioned-object-get-tags" + version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("Version 1"), + Key: aws.String(objectKey), + Bucket: aws.String(bucketName), + Tagging: aws.String("v=1&stage=dev"), + }) + require.NoError(t, err) + + versionId1 := *version1Resp.VersionId + time.Sleep(50 * time.Millisecond) + + version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("Version 2"), + Key: aws.String(objectKey), + Bucket: aws.String(bucketName), + Tagging: aws.String("v=2&stage=prod"), + }) + require.NoError(t, err) + + versionId2 := *version2Resp.VersionId + + // Get tags from specific versions + tagResp1, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + }) + require.NoError(t, err, "Should be able to get tags from version 1") + assert.Len(t, tagResp1.TagSet, 2, "Version 1 should have 2 tags") + tagMap1 := make(map[string]string) + for _, tag := range tagResp1.TagSet { + tagMap1[*tag.Key] = *tag.Value + } + assert.Equal(t, "1", tagMap1["v"], "Version 1 should have v=1") + assert.Equal(t, "dev", tagMap1["stage"], "Version 1 should have stage=dev") + + tagResp2, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + }) + require.NoError(t, err, "Should be able to get tags from version 2") + assert.Len(t, tagResp2.TagSet, 2, "Version 2 should have 2 tags") + tagMap2 := make(map[string]string) + for _, tag := range tagResp2.TagSet { + tagMap2[*tag.Key] = *tag.Value + } + assert.Equal(t, "2", tagMap2["v"], "Version 2 should have v=2") + assert.Equal(t, "prod", tagMap2["stage"], "Version 2 should have stage=prod") + + // Get tags from latest version (should be version 2) + tagRespLatest, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + }) + require.NoError(t, err, "Should be able to get tags from latest version") + assert.Len(t, tagRespLatest.TagSet, 2, "Latest version should have 2 tags") + tagMapLatest := make(map[string]string) + for _, tag := range tagRespLatest.TagSet { + tagMapLatest[*tag.Key] = *tag.Value + } + assert.Equal(t, "2", tagMapLatest["v"], "Latest version should have v=2") + assert.Equal(t, "prod", tagMapLatest["stage"], "Latest version should have stage=prod") +} + +// TestModifyTagsOnVersionedObject tests changing tags on different versions independently +func TestModifyTagsOnVersionedObject(t *testing.T) { + client := getS3Client(t) + bucketName := createVersionedTestBucket(t, client) + defer cleanupTestBucket(t, client, bucketName) + + // Create version 1 + objectKey := "versioned-modify-tags" + version1Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("Version 1"), + Key: aws.String(objectKey), + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + + versionId1 := *version1Resp.VersionId + time.Sleep(50 * time.Millisecond) + + // Create version 2 + version2Resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("Version 2"), + Key: aws.String(objectKey), + Bucket: aws.String(bucketName), + }) + require.NoError(t, err) + + versionId2 := *version2Resp.VersionId + + // Add tags to version 1 + _, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + Tagging: &types.Tagging{ + TagSet: []types.Tag{ + { + Key: aws.String("status"), + Value: aws.String("old"), + }, + }, + }, + }) + require.NoError(t, err) + + // Modify tags on version 1 (replace status and add new tag) + _, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + Tagging: &types.Tagging{ + TagSet: []types.Tag{ + { + Key: aws.String("status"), + Value: aws.String("archived"), + }, + { + Key: aws.String("archived-date"), + Value: aws.String("2024-01-01"), + }, + }, + }, + }) + require.NoError(t, err) + + // Add tags to version 2 + _, err = client.PutObjectTagging(context.TODO(), &s3.PutObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + Tagging: &types.Tagging{ + TagSet: []types.Tag{ + { + Key: aws.String("status"), + Value: aws.String("current"), + }, + }, + }, + }) + require.NoError(t, err) + + // Verify final state + tagResp1, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId1), + }) + require.NoError(t, err) + assert.Len(t, tagResp1.TagSet, 2, "Version 1 should have 2 tags after modification") + + tagResp2, err := client.GetObjectTagging(context.TODO(), &s3.GetObjectTaggingInput{ + Bucket: aws.String(bucketName), + Key: aws.String(objectKey), + VersionId: aws.String(versionId2), + }) + require.NoError(t, err) + assert.Len(t, tagResp2.TagSet, 1, "Version 2 should have 1 tag") +} + +// createVersionedTestBucket creates a test bucket with versioning enabled +func createVersionedTestBucket(t *testing.T, client *s3.Client) string { + bucketName := createTestBucket(t, client) + + // Enable versioning on the bucket + _, err := client.PutBucketVersioning(context.TODO(), &s3.PutBucketVersioningInput{ + Bucket: aws.String(bucketName), + VersioningConfiguration: &types.VersioningConfiguration{ + Status: types.BucketVersioningStatusEnabled, + }, + }) + require.NoError(t, err, "Should be able to enable versioning") + + // Wait for versioning configuration to be applied + time.Sleep(100 * time.Millisecond) + + return bucketName +} diff --git a/test/s3/versioning/Makefile b/test/s3/versioning/Makefile index 08a18fd96..7e939f90e 100644 --- a/test/s3/versioning/Makefile +++ b/test/s3/versioning/Makefile @@ -43,13 +43,13 @@ build-weed: @echo "Building SeaweedFS binary..." @cd ../../../weed && go build -o weed_binary . @chmod +x $(WEED_BINARY) - @echo "✅ SeaweedFS binary built at $(WEED_BINARY)" + @echo "OK SeaweedFS binary built at $(WEED_BINARY)" check-deps: build-weed @echo "Checking dependencies..." - @echo "🔍 DEBUG: Checking Go installation..." + @echo "DEBUG: Checking Go installation..." @command -v go >/dev/null 2>&1 || (echo "Go is required but not installed" && exit 1) - @echo "🔍 DEBUG: Go version: $$(go version)" + @echo "DEBUG: Go version: $$(go version)" @echo "🔍 DEBUG: Checking binary at $(WEED_BINARY)..." @test -f $(WEED_BINARY) || (echo "SeaweedFS binary not found at $(WEED_BINARY)" && exit 1) @echo "🔍 DEBUG: Binary size: $$(ls -lh $(WEED_BINARY) | awk '{print $$5}')" @@ -57,7 +57,7 @@ check-deps: build-weed @echo "🔍 DEBUG: Checking Go module dependencies..." @go list -m github.com/aws/aws-sdk-go-v2 >/dev/null 2>&1 || (echo "AWS SDK Go v2 not found. Run 'go mod tidy'." && exit 1) @go list -m github.com/stretchr/testify >/dev/null 2>&1 || (echo "Testify not found. Run 'go mod tidy'." && exit 1) - @echo "✅ All dependencies are available" + @echo "OK All dependencies are available" # Start SeaweedFS server for testing start-server: check-deps @@ -81,21 +81,11 @@ start-server: check-deps @echo "🔍 DEBUG: Creating volume directory..." @mkdir -p ./test-volume-data @echo "🔍 DEBUG: Launching SeaweedFS server in background..." - @echo "🔍 DEBUG: Command: $(WEED_BINARY) server -debug -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true -s3.config=../../../docker/compose/s3.json -filer -filer.maxMB=64 -master.volumeSizeLimitMB=50 -master.peers=none -volume.max=100 -dir=./test-volume-data -volume.preStopSeconds=1 -metricsPort=9324" - @$(WEED_BINARY) server \ - -debug \ - -s3 \ - -s3.port=$(S3_PORT) \ - -s3.allowDeleteBucketNotEmpty=true \ - -s3.config=../../../docker/compose/s3.json \ - -filer \ - -filer.maxMB=64 \ - -master.volumeSizeLimitMB=50 \ - -master.peers=none \ - -volume.max=100 \ + @echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=./test-volume-data -s3.port=$(S3_PORT) -s3.config=../../../docker/compose/s3.json" + @$(WEED_BINARY) mini \ -dir=./test-volume-data \ - -volume.preStopSeconds=1 \ - -metricsPort=9324 \ + -s3.port=$(S3_PORT) \ + -s3.config=../../../docker/compose/s3.json \ > weed-test.log 2>&1 & echo $$! > weed-server.pid @echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')" @echo "🔍 DEBUG: Checking if PID is still running..." @@ -222,13 +212,13 @@ test-with-server: start-server test-versioning-with-configs: check-deps @echo "Testing with different S3 configurations..." @echo "Testing with empty folder allowed..." - @$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid + @$(WEED_BINARY) mini -s3.port=$(S3_PORT) > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid @sleep 5 @go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true @if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi @sleep 2 @echo "Testing with delete bucket not empty disabled..." - @$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -master.peers=none -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid + @$(WEED_BINARY) mini -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid @sleep 5 @go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true @if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi @@ -271,19 +261,14 @@ debug-server: @echo "Starting SeaweedFS server in debug mode..." @$(MAKE) stop-server @mkdir -p ./test-volume-data - @$(WEED_BINARY) server \ + @$(WEED_BINARY) mini \ -debug \ - -s3 \ -s3.port=$(S3_PORT) \ -s3.allowDeleteBucketNotEmpty=true \ -s3.config=../../../docker/compose/s3.json \ - -filer \ -filer.maxMB=16 \ - -master.volumeSizeLimitMB=50 \ -master.peers=none \ - -volume.max=100 \ -dir=./test-volume-data \ - -volume.preStopSeconds=1 \ -metricsPort=9324 # Run a single test for debugging @@ -320,24 +305,24 @@ health-check: # Simple server start without process cleanup (for CI troubleshooting) start-server-simple: check-deps @echo "Starting SeaweedFS server (simple mode)..." - @$(WEED_BINARY) server \ - -debug \ + @$(WEED_BINARY) mini \ + -dir=$(SERVER_DIR) \ -s3 \ -s3.port=$(S3_PORT) \ - -s3.allowDeleteBucketNotEmpty=true \ - -s3.config=../../../docker/compose/s3.json \ - -filer \ - -filer.maxMB=64 \ - -master.volumeSizeLimitMB=50 \ - -master.peers=none \ - -volume.max=100 \ - -volume.preStopSeconds=1 \ - -metricsPort=9324 \ - > weed-test.log 2>&1 & echo $$! > weed-server.pid - @echo "Server PID: $$(cat weed-server.pid)" - @echo "Waiting for server to start..." - @sleep 10 - @curl -s http://localhost:$(S3_PORT) >/dev/null 2>&1 && echo "✅ Server started successfully" || echo "❌ Server failed to start" + -s3.config=$(S3_CONFIG) \ + > weed-server.log 2>&1 & \ + echo $$! > weed-server.pid + + @echo "Waiting for S3 server to be ready..." + @for i in $$(seq 1 30); do \ + if echo | nc -z localhost $(S3_PORT); then \ + echo "S3 server is ready!"; \ + exit 0; \ + fi; \ + sleep 1; \ + done; \ + echo "S3 server failed to start"; \ + exit 1 # Simple test run without server management test-versioning-simple: check-deps diff --git a/weed/admin/Makefile b/weed/admin/Makefile index b79ddc1ab..605545d3b 100644 --- a/weed/admin/Makefile +++ b/weed/admin/Makefile @@ -160,6 +160,4 @@ $(WEED_BINARY): $(TEMPL_GO_FILES) $(GO_FILES) # Auto-generate templ files when .templ files change %_templ.go: %.templ @echo "Regenerating $@ from $<" - @templ generate - -.PHONY: $(TEMPL_GO_FILES) \ No newline at end of file + @templ generate diff --git a/weed/admin/dash/admin_server.go b/weed/admin/dash/admin_server.go index 549a431bd..34ce82329 100644 --- a/weed/admin/dash/admin_server.go +++ b/weed/admin/dash/admin_server.go @@ -7,6 +7,7 @@ import ( "net/http" "sort" "strconv" + "strings" "time" "github.com/gin-gonic/gin" @@ -340,7 +341,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) { } // Get versioning, object lock, and owner information from extended attributes - versioningEnabled := false + versioningStatus := "" objectLockEnabled := false objectLockMode := "" var objectLockDuration int32 = 0 @@ -348,7 +349,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) { if resp.Entry.Extended != nil { // Use shared utility to extract versioning information - versioningEnabled = extractVersioningFromEntry(resp.Entry) + versioningStatus = extractVersioningFromEntry(resp.Entry) // Use shared utility to extract Object Lock information objectLockEnabled, objectLockMode, objectLockDuration = extractObjectLockInfoFromEntry(resp.Entry) @@ -367,7 +368,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) { LastModified: time.Unix(resp.Entry.Attributes.Mtime, 0), Quota: quota, QuotaEnabled: quotaEnabled, - VersioningEnabled: versioningEnabled, + VersioningStatus: versioningStatus, ObjectLockEnabled: objectLockEnabled, ObjectLockMode: objectLockMode, ObjectLockDuration: objectLockDuration, @@ -430,7 +431,7 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error details.Bucket.QuotaEnabled = quotaEnabled // Get versioning, object lock, and owner information from extended attributes - versioningEnabled := false + versioningStatus := "" objectLockEnabled := false objectLockMode := "" var objectLockDuration int32 = 0 @@ -438,7 +439,7 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error if bucketResp.Entry.Extended != nil { // Use shared utility to extract versioning information - versioningEnabled = extractVersioningFromEntry(bucketResp.Entry) + versioningStatus = extractVersioningFromEntry(bucketResp.Entry) // Use shared utility to extract Object Lock information objectLockEnabled, objectLockMode, objectLockDuration = extractObjectLockInfoFromEntry(bucketResp.Entry) @@ -449,7 +450,7 @@ func (s *AdminServer) GetBucketDetails(bucketName string) (*BucketDetails, error } } - details.Bucket.VersioningEnabled = versioningEnabled + details.Bucket.VersioningStatus = versioningStatus details.Bucket.ObjectLockEnabled = objectLockEnabled details.Bucket.ObjectLockMode = objectLockMode details.Bucket.ObjectLockDuration = objectLockDuration @@ -491,6 +492,45 @@ func (s *AdminServer) listBucketObjects(client filer_pb.SeaweedFilerClient, buck entry := resp.Entry if entry.IsDirectory { + // Check if this is a .versions directory (represents a versioned object) + if strings.HasSuffix(entry.Name, ".versions") { + // This directory represents an object, add it as an object without the .versions suffix + objectName := strings.TrimSuffix(entry.Name, ".versions") + objectKey := objectName + if directory != bucketBasePath { + relativePath := directory[len(bucketBasePath)+1:] + objectKey = fmt.Sprintf("%s/%s", relativePath, objectName) + } + + // Extract latest version metadata from extended attributes + var size int64 = 0 + var mtime int64 = entry.Attributes.Mtime + if entry.Extended != nil { + // Get size of latest version + if sizeBytes, ok := entry.Extended[s3_constants.ExtLatestVersionSizeKey]; ok && len(sizeBytes) == 8 { + size = int64(util.BytesToUint64(sizeBytes)) + } + // Get mtime of latest version + if mtimeBytes, ok := entry.Extended[s3_constants.ExtLatestVersionMtimeKey]; ok && len(mtimeBytes) == 8 { + mtime = int64(util.BytesToUint64(mtimeBytes)) + } + } + + obj := S3Object{ + Key: objectKey, + Size: size, + LastModified: time.Unix(mtime, 0), + ETag: "", + StorageClass: "STANDARD", + } + + details.Objects = append(details.Objects, obj) + details.TotalCount++ + details.TotalSize += size + // Don't recurse into .versions directories + continue + } + // Recursively list subdirectories subDir := fmt.Sprintf("%s/%s", directory, entry.Name) err := s.listBucketObjects(client, bucketBasePath, subDir, "", details) @@ -583,7 +623,7 @@ func (s *AdminServer) DeleteS3Bucket(bucketName string) error { } // GetObjectStoreUsers retrieves object store users from identity.json -func (s *AdminServer) GetObjectStoreUsers() ([]ObjectStoreUser, error) { +func (s *AdminServer) GetObjectStoreUsers(ctx context.Context) ([]ObjectStoreUser, error) { s3cfg := &iam_pb.S3ApiConfiguration{} // Load IAM configuration from filer @@ -616,6 +656,11 @@ func (s *AdminServer) GetObjectStoreUsers() ([]ObjectStoreUser, error) { continue } + // Skip service accounts - they should not be parent users + if strings.HasPrefix(identity.Name, serviceAccountPrefix) { + continue + } + user := ObjectStoreUser{ Username: identity.Name, Permissions: identity.Actions, @@ -1902,9 +1947,8 @@ func extractObjectLockInfoFromEntry(entry *filer_pb.Entry) (bool, string, int32) } // Function to extract versioning information from bucket entry using shared utilities -func extractVersioningFromEntry(entry *filer_pb.Entry) bool { - enabled, _ := s3api.LoadVersioningFromExtended(entry) - return enabled +func extractVersioningFromEntry(entry *filer_pb.Entry) string { + return s3api.GetVersioningStatus(entry) } // GetConfigPersistence returns the config persistence manager diff --git a/weed/admin/dash/auth_middleware.go b/weed/admin/dash/auth_middleware.go index 87da65659..5da81481a 100644 --- a/weed/admin/dash/auth_middleware.go +++ b/weed/admin/dash/auth_middleware.go @@ -1,6 +1,7 @@ package dash import ( + "crypto/subtle" "net/http" "github.com/gin-contrib/sessions" @@ -25,17 +26,31 @@ func (s *AdminServer) ShowLogin(c *gin.Context) { } // HandleLogin handles login form submission -func (s *AdminServer) HandleLogin(username, password string) gin.HandlerFunc { +func (s *AdminServer) HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword string) gin.HandlerFunc { return func(c *gin.Context) { loginUsername := c.PostForm("username") loginPassword := c.PostForm("password") - if loginUsername == username && loginPassword == password { + var role string + var authenticated bool + + // Check admin credentials + if adminPassword != "" && loginUsername == adminUser && subtle.ConstantTimeCompare([]byte(loginPassword), []byte(adminPassword)) == 1 { + role = "admin" + authenticated = true + } else if readOnlyPassword != "" && loginUsername == readOnlyUser && subtle.ConstantTimeCompare([]byte(loginPassword), []byte(readOnlyPassword)) == 1 { + // Check read-only credentials + role = "readonly" + authenticated = true + } + + if authenticated { session := sessions.Default(c) // Clear any existing invalid session data before setting new values session.Clear() session.Set("authenticated", true) session.Set("username", loginUsername) + session.Set("role", role) if err := session.Save(); err != nil { // Log the detailed error server-side for diagnostics glog.Errorf("Failed to save session for user %s: %v", loginUsername, err) diff --git a/weed/admin/dash/bucket_management.go b/weed/admin/dash/bucket_management.go index eb99e9fa4..7104aa8c6 100644 --- a/weed/admin/dash/bucket_management.go +++ b/weed/admin/dash/bucket_management.go @@ -125,6 +125,12 @@ func (s *AdminServer) CreateBucket(c *gin.Context) { // Convert quota to bytes quotaBytes := convertQuotaToBytes(req.QuotaSize, req.QuotaUnit) + // Validate quota: if enabled, size must be greater than 0 + if req.QuotaEnabled && quotaBytes <= 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "Quota size must be greater than 0 when quota is enabled"}) + return + } + // Sanitize owner: trim whitespace and enforce max length owner := strings.TrimSpace(req.Owner) if len(owner) > MaxOwnerNameLength { @@ -466,16 +472,19 @@ func (s *AdminServer) CreateS3BucketWithObjectLock(bucketName string, quotaBytes // Handle Object Lock configuration using shared utilities if objectLockEnabled { var duration int32 = 0 + var mode string = "" + if setDefaultRetention { // Validate Object Lock parameters only when setting default retention if err := s3api.ValidateObjectLockParameters(objectLockEnabled, objectLockMode, objectLockDuration); err != nil { return fmt.Errorf("invalid Object Lock parameters: %w", err) } duration = objectLockDuration + mode = objectLockMode } // Create Object Lock configuration using shared utility - objectLockConfig := s3api.CreateObjectLockConfigurationFromParams(objectLockEnabled, objectLockMode, duration) + objectLockConfig := s3api.CreateObjectLockConfigurationFromParams(objectLockEnabled, mode, duration) // Store Object Lock configuration in extended attributes using shared utility if err := s3api.StoreObjectLockConfigurationInExtended(bucketEntry, objectLockConfig); err != nil { diff --git a/weed/admin/dash/file_browser_data.go b/weed/admin/dash/file_browser_data.go index 6bb30c469..6e6e44c9d 100644 --- a/weed/admin/dash/file_browser_data.go +++ b/weed/admin/dash/file_browser_data.go @@ -2,8 +2,7 @@ package dash import ( "context" - "path/filepath" - "sort" + "path" "strings" "time" @@ -34,34 +33,49 @@ type BreadcrumbItem struct { // FileBrowserData contains all data needed for the file browser view type FileBrowserData struct { - Username string `json:"username"` - CurrentPath string `json:"current_path"` - ParentPath string `json:"parent_path"` - Breadcrumbs []BreadcrumbItem `json:"breadcrumbs"` - Entries []FileEntry `json:"entries"` - TotalEntries int `json:"total_entries"` - TotalSize int64 `json:"total_size"` - LastUpdated time.Time `json:"last_updated"` - IsBucketPath bool `json:"is_bucket_path"` - BucketName string `json:"bucket_name"` + Username string `json:"username"` + CurrentPath string `json:"current_path"` + ParentPath string `json:"parent_path"` + Breadcrumbs []BreadcrumbItem `json:"breadcrumbs"` + Entries []FileEntry `json:"entries"` + + LastUpdated time.Time `json:"last_updated"` + IsBucketPath bool `json:"is_bucket_path"` + BucketName string `json:"bucket_name"` + // Pagination fields + PageSize int `json:"page_size"` + HasNextPage bool `json:"has_next_page"` + LastFileName string `json:"last_file_name"` // Cursor for next page + CurrentLastFileName string `json:"current_last_file_name"` // Cursor from current request (for page size changes) } -// GetFileBrowser retrieves file browser data for a given path -func (s *AdminServer) GetFileBrowser(path string) (*FileBrowserData, error) { - if path == "" { - path = "/" +// GetFileBrowser retrieves file browser data for a given path with cursor-based pagination +func (s *AdminServer) GetFileBrowser(dir string, lastFileName string, pageSize int) (*FileBrowserData, error) { + if dir == "" { + dir = "/" + } + + // Set defaults for pagination + if pageSize < 1 { + pageSize = 20 // Default page size } var entries []FileEntry - var totalSize int64 - // Get directory listing from filer + // Fetch entries using cursor-based pagination + // We fetch pageSize+1 to determine if there's a next page + fetchLimit := pageSize + 1 + var fetchedCount int + var lastEntryName string + err := s.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { + // Fetch entries starting from the cursor (lastFileName) stream, err := client.ListEntries(context.Background(), &filer_pb.ListEntriesRequest{ - Directory: path, + Directory: dir, Prefix: "", - Limit: 1000, - InclusiveStartFrom: false, + Limit: uint32(fetchLimit), + StartFromFileName: lastFileName, + InclusiveStartFrom: false, // Don't include the cursor file itself }) if err != nil { return err @@ -81,101 +95,102 @@ func (s *AdminServer) GetFileBrowser(path string) (*FileBrowserData, error) { continue } - fullPath := path - if !strings.HasSuffix(fullPath, "/") { - fullPath += "/" - } - fullPath += entry.Name + fetchedCount++ - var modTime time.Time - if entry.Attributes != nil && entry.Attributes.Mtime > 0 { - modTime = time.Unix(entry.Attributes.Mtime, 0) - } + // Only add entries up to pageSize (the +1 is just to check for next page) + if fetchedCount <= pageSize { + fullPath := path.Join(dir, entry.Name) - var mode string - var uid, gid uint32 - var size int64 - var replication, collection string - var ttlSec int32 - - if entry.Attributes != nil { - mode = FormatFileMode(entry.Attributes.FileMode) - uid = entry.Attributes.Uid - gid = entry.Attributes.Gid - size = int64(entry.Attributes.FileSize) - ttlSec = entry.Attributes.TtlSec - } - - // Get replication and collection from entry extended attributes or chunks - if entry.Extended != nil { - if repl, ok := entry.Extended["replication"]; ok { - replication = string(repl) + var modTime time.Time + if entry.Attributes != nil && entry.Attributes.Mtime > 0 { + modTime = time.Unix(entry.Attributes.Mtime, 0) } - if coll, ok := entry.Extended["collection"]; ok { - collection = string(coll) + + var mode string + var uid, gid uint32 + var size int64 + var replication, collection string + var ttlSec int32 + + if entry.Attributes != nil { + mode = FormatFileMode(entry.Attributes.FileMode) + uid = entry.Attributes.Uid + gid = entry.Attributes.Gid + size = int64(entry.Attributes.FileSize) + ttlSec = entry.Attributes.TtlSec } - } - // Determine MIME type based on file extension - mime := "application/octet-stream" - if entry.IsDirectory { - mime = "inode/directory" - } else { - ext := strings.ToLower(filepath.Ext(entry.Name)) - switch ext { - case ".txt", ".log": - mime = "text/plain" - case ".html", ".htm": - mime = "text/html" - case ".css": - mime = "text/css" - case ".js": - mime = "application/javascript" - case ".json": - mime = "application/json" - case ".xml": - mime = "application/xml" - case ".pdf": - mime = "application/pdf" - case ".jpg", ".jpeg": - mime = "image/jpeg" - case ".png": - mime = "image/png" - case ".gif": - mime = "image/gif" - case ".svg": - mime = "image/svg+xml" - case ".mp4": - mime = "video/mp4" - case ".mp3": - mime = "audio/mpeg" - case ".zip": - mime = "application/zip" - case ".tar": - mime = "application/x-tar" - case ".gz": - mime = "application/gzip" + // Get replication and collection from entry extended attributes + if entry.Extended != nil { + if repl, ok := entry.Extended["replication"]; ok { + replication = string(repl) + } + if coll, ok := entry.Extended["collection"]; ok { + collection = string(coll) + } } - } - fileEntry := FileEntry{ - Name: entry.Name, - FullPath: fullPath, - IsDirectory: entry.IsDirectory, - Size: size, - ModTime: modTime, - Mode: mode, - Uid: uid, - Gid: gid, - Mime: mime, - Replication: replication, - Collection: collection, - TtlSec: ttlSec, - } + // Determine MIME type based on file extension + mime := "application/octet-stream" + if entry.IsDirectory { + mime = "inode/directory" + } else { + ext := strings.ToLower(path.Ext(entry.Name)) + switch ext { + case ".txt", ".log": + mime = "text/plain" + case ".html", ".htm": + mime = "text/html" + case ".css": + mime = "text/css" + case ".js": + mime = "application/javascript" + case ".json": + mime = "application/json" + case ".xml": + mime = "application/xml" + case ".pdf": + mime = "application/pdf" + case ".jpg", ".jpeg": + mime = "image/jpeg" + case ".png": + mime = "image/png" + case ".gif": + mime = "image/gif" + case ".svg": + mime = "image/svg+xml" + case ".mp4": + mime = "video/mp4" + case ".mp3": + mime = "audio/mpeg" + case ".zip": + mime = "application/zip" + case ".tar": + mime = "application/x-tar" + case ".gz": + mime = "application/gzip" + } + } + + fileEntry := FileEntry{ + Name: entry.Name, + FullPath: fullPath, + IsDirectory: entry.IsDirectory, + Size: size, + ModTime: modTime, + Mode: mode, + Uid: uid, + Gid: gid, + Mime: mime, + Replication: replication, + Collection: collection, + TtlSec: ttlSec, + } + + entries = append(entries, fileEntry) + + lastEntryName = entry.Name - entries = append(entries, fileEntry) - if !entry.IsDirectory { - totalSize += size } } @@ -186,21 +201,16 @@ func (s *AdminServer) GetFileBrowser(path string) (*FileBrowserData, error) { return nil, err } - // Sort entries: directories first, then files, both alphabetically - sort.Slice(entries, func(i, j int) bool { - if entries[i].IsDirectory != entries[j].IsDirectory { - return entries[i].IsDirectory - } - return strings.ToLower(entries[i].Name) < strings.ToLower(entries[j].Name) - }) + // Determine if there's a next page + hasNextPage := fetchedCount > pageSize // Generate breadcrumbs - breadcrumbs := s.generateBreadcrumbs(path) + breadcrumbs := s.generateBreadcrumbs(dir) // Calculate parent path parentPath := "/" - if path != "/" { - parentPath = filepath.Dir(path) + if dir != "/" { + parentPath = path.Dir(dir) if parentPath == "." { parentPath = "/" } @@ -209,29 +219,33 @@ func (s *AdminServer) GetFileBrowser(path string) (*FileBrowserData, error) { // Check if this is a bucket path isBucketPath := false bucketName := "" - if strings.HasPrefix(path, "/buckets/") { + if strings.HasPrefix(dir, "/buckets/") { isBucketPath = true - pathParts := strings.Split(strings.Trim(path, "/"), "/") + pathParts := strings.Split(strings.Trim(dir, "/"), "/") if len(pathParts) >= 2 { bucketName = pathParts[1] } } return &FileBrowserData{ - CurrentPath: path, - ParentPath: parentPath, - Breadcrumbs: breadcrumbs, - Entries: entries, - TotalEntries: len(entries), - TotalSize: totalSize, + CurrentPath: dir, + ParentPath: parentPath, + Breadcrumbs: breadcrumbs, + Entries: entries, + LastUpdated: time.Now(), IsBucketPath: isBucketPath, BucketName: bucketName, + // Pagination metadata + PageSize: pageSize, + HasNextPage: hasNextPage, + LastFileName: lastEntryName, // Store for next page navigation + CurrentLastFileName: lastFileName, // Store input cursor for page size changes }, nil } // generateBreadcrumbs creates breadcrumb navigation for the current path -func (s *AdminServer) generateBreadcrumbs(path string) []BreadcrumbItem { +func (s *AdminServer) generateBreadcrumbs(dir string) []BreadcrumbItem { var breadcrumbs []BreadcrumbItem // Always start with root @@ -240,12 +254,12 @@ func (s *AdminServer) generateBreadcrumbs(path string) []BreadcrumbItem { Path: "/", }) - if path == "/" { + if dir == "/" { return breadcrumbs } // Split path and build breadcrumbs - parts := strings.Split(strings.Trim(path, "/"), "/") + parts := strings.Split(strings.Trim(dir, "/"), "/") currentPath := "" for _, part := range parts { @@ -258,7 +272,7 @@ func (s *AdminServer) generateBreadcrumbs(path string) []BreadcrumbItem { displayName := part if len(breadcrumbs) == 1 && part == "buckets" { displayName = "Object Store Buckets" - } else if len(breadcrumbs) == 2 && strings.HasPrefix(path, "/buckets/") { + } else if len(breadcrumbs) == 2 && strings.HasPrefix(dir, "/buckets/") { displayName = "📦 " + part // Add bucket icon to bucket name } diff --git a/weed/admin/dash/file_browser_data_test.go b/weed/admin/dash/file_browser_data_test.go new file mode 100644 index 000000000..0605735af --- /dev/null +++ b/weed/admin/dash/file_browser_data_test.go @@ -0,0 +1,502 @@ +package dash + +import ( + "path" + "strings" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/util" +) + +// TestGenerateBreadcrumbs tests the actual breadcrumb generation function +// from the production code with various path scenarios +func TestGenerateBreadcrumbs(t *testing.T) { + s := &AdminServer{} + + tests := []struct { + name string + path string + expected []BreadcrumbItem + }{ + { + name: "root path", + path: "/", + expected: []BreadcrumbItem{ + {Name: "Root", Path: "/"}, + }, + }, + { + name: "simple path", + path: "/folder", + expected: []BreadcrumbItem{ + {Name: "Root", Path: "/"}, + {Name: "folder", Path: "/folder"}, + }, + }, + { + name: "nested path", + path: "/folder/subfolder", + expected: []BreadcrumbItem{ + {Name: "Root", Path: "/"}, + {Name: "folder", Path: "/folder"}, + {Name: "subfolder", Path: "/folder/subfolder"}, + }, + }, + { + name: "bucket path", + path: "/buckets/mybucket", + expected: []BreadcrumbItem{ + {Name: "Root", Path: "/"}, + {Name: "Object Store Buckets", Path: "/buckets"}, + {Name: "📦 mybucket", Path: "/buckets/mybucket"}, + }, + }, + { + name: "bucket nested path", + path: "/buckets/mybucket/folder", + expected: []BreadcrumbItem{ + {Name: "Root", Path: "/"}, + {Name: "Object Store Buckets", Path: "/buckets"}, + {Name: "📦 mybucket", Path: "/buckets/mybucket"}, + {Name: "folder", Path: "/buckets/mybucket/folder"}, + }, + }, + { + name: "path with trailing slash", + path: "/folder/", + expected: []BreadcrumbItem{ + {Name: "Root", Path: "/"}, + {Name: "folder", Path: "/folder"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Call the actual production function + result := s.generateBreadcrumbs(tt.path) + + if len(result) != len(tt.expected) { + t.Errorf("expected %d breadcrumbs, got %d", len(tt.expected), len(result)) + return + } + + for i, crumb := range result { + if crumb.Name != tt.expected[i].Name { + t.Errorf("breadcrumb %d: expected name %q, got %q", i, tt.expected[i].Name, crumb.Name) + } + if crumb.Path != tt.expected[i].Path { + t.Errorf("breadcrumb %d: expected path %q, got %q", i, tt.expected[i].Path, crumb.Path) + } + } + }) + } +} + +// TestPathHandlingWithForwardSlashes verifies that the production code +// correctly handles paths with forward slashes (not OS-specific backslashes) +func TestPathHandlingWithForwardSlashes(t *testing.T) { + tests := []struct { + name string + path string + hasSlash bool + }{ + { + name: "root", + path: "/", + hasSlash: true, + }, + { + name: "single level", + path: "/test", + hasSlash: true, + }, + { + name: "multiple levels", + path: "/a/b/c", + hasSlash: true, + }, + { + name: "bucket path", + path: "/buckets/mybucket/file", + hasSlash: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Verify no backslashes appear in paths (which would be wrong on Windows) + if strings.Contains(tt.path, "\\") { + t.Errorf("path contains backslash: %q", tt.path) + } + + // Verify forward slashes are used + if tt.hasSlash && !strings.Contains(tt.path, "/") { + t.Errorf("path should contain forward slash: %q", tt.path) + } + }) + } +} + +// TestParentPathCalculationLogic verifies that parent path calculation +// uses path.Dir semantics (forward slashes), not filepath.Dir (OS-specific) +func TestParentPathCalculationLogic(t *testing.T) { + tests := []struct { + name string + currentDir string + expected string + }{ + { + name: "root path", + currentDir: "/", + expected: "/", + }, + { + name: "single level", + currentDir: "/folder", + expected: "/", + }, + { + name: "two levels", + currentDir: "/folder/subfolder", + expected: "/folder", + }, + { + name: "deep nesting", + currentDir: "/a/b/c/d", + expected: "/a/b/c", + }, + { + name: "bucket root", + currentDir: "/buckets", + expected: "/", + }, + { + name: "bucket directory", + currentDir: "/buckets/mybucket", + expected: "/buckets", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Verify using path.Dir (the correct approach for URLs) + // This demonstrates the expected behavior + parentPath := "/" + if tt.currentDir != "/" { + // path.Dir always uses forward slashes + parentPath = path.Dir(tt.currentDir) + if parentPath == "." { + parentPath = "/" + } + } + + if parentPath != tt.expected { + t.Errorf("expected parent %q, got %q for %q", tt.expected, parentPath, tt.currentDir) + } + + // Verify no backslashes in the result + if strings.Contains(parentPath, "\\") { + t.Errorf("parent path contains backslash: %q", parentPath) + } + }) + } +} + +// TestFileExtensionHandlingLogic verifies that file extensions are correctly +// identified using path semantics (always forward slashes) +func TestFileExtensionHandlingLogic(t *testing.T) { + tests := []struct { + filename string + expected string + }{ + {"file.txt", ".txt"}, + {"file.log", ".log"}, + {"archive.tar.gz", ".gz"}, + {"image.jpg", ".jpg"}, + {"document.pdf", ".pdf"}, + {"data.json", ".json"}, + {"noextension", ""}, + {".hidden", ".hidden"}, + {"file.TXT", ".txt"}, + {"file.JPG", ".jpg"}, + } + + for _, tt := range tests { + t.Run(tt.filename, func(t *testing.T) { + // Verify using path.Ext + strings.ToLower (the correct approach) + ext := strings.ToLower(path.Ext(tt.filename)) + if ext != tt.expected { + t.Errorf("expected extension %q for %q, got %q", tt.expected, tt.filename, ext) + } + }) + } +} + +// TestBucketPathDetectionLogic verifies bucket path detection logic +func TestBucketPathDetectionLogic(t *testing.T) { + tests := []struct { + name string + path string + isBucket bool + expectedName string + }{ + { + name: "root is not a bucket path", + path: "/", + isBucket: false, + expectedName: "", + }, + { + name: "buckets root", + path: "/buckets/", + isBucket: true, + expectedName: "", + }, + { + name: "single bucket", + path: "/buckets/mybucket", + isBucket: true, + expectedName: "mybucket", + }, + { + name: "bucket with nested path", + path: "/buckets/mybucket/folder/file", + isBucket: true, + expectedName: "mybucket", + }, + { + name: "non-bucket path", + path: "/data/folder", + isBucket: false, + expectedName: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Verify the bucket path detection logic + isBucketPath := false + bucketName := "" + if strings.HasPrefix(tt.path, "/buckets/") { + isBucketPath = true + pathParts := strings.Split(strings.Trim(tt.path, "/"), "/") + if len(pathParts) >= 2 { + bucketName = pathParts[1] + } + } + + if isBucketPath != tt.isBucket { + t.Errorf("expected isBucketPath=%v, got %v for path %q", tt.isBucket, isBucketPath, tt.path) + } + + if bucketName != tt.expectedName { + t.Errorf("expected bucket name %q, got %q for path %q", tt.expectedName, bucketName, tt.path) + } + }) + } +} + +// TestPathJoinHandlesEdgeCases verifies that path.Join handles edge cases +// properly for URL path construction (unlike filepath.Join which is OS-specific) +func TestPathJoinHandlesEdgeCases(t *testing.T) { + tests := []struct { + testName string + dir string + filename string + expected string + }{ + { + testName: "root directory", + dir: "/", + filename: "file.txt", + expected: "/file.txt", + }, + { + testName: "simple directory", + dir: "/folder", + filename: "file.txt", + expected: "/folder/file.txt", + }, + { + testName: "nested directory", + dir: "/a/b/c", + filename: "file.txt", + expected: "/a/b/c/file.txt", + }, + { + testName: "handles trailing slash", + dir: "/folder/", + filename: "file.txt", + expected: "/folder/file.txt", + }, + { + testName: "handles empty name", + dir: "/folder", + filename: "", + expected: "/folder", + }, + } + + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + // Verify path.Join behavior for URL paths + result := path.Join(tt.dir, tt.filename) + if result != tt.expected { + t.Errorf("path.Join(%q, %q) = %q, expected %q", tt.dir, tt.filename, result, tt.expected) + } + + // Verify no backslashes in the result + if strings.Contains(result, "\\") { + t.Errorf("result contains backslash: %q", result) + } + }) + } +} + +// TestWindowsPathNormalizationBehavior validates that Windows-style paths +// are correctly converted to forward slashes for URL compatibility. +// This test verifies the actual util.CleanWindowsPath() function used in +// the ShowFileBrowser handler. +func TestWindowsPathNormalizationBehavior(t *testing.T) { + tests := []struct { + name string + windowsPath string + expectedNormPath string + }{ + { + name: "backslash separator", + windowsPath: "\\folder\\subfolder", + expectedNormPath: "/folder/subfolder", + }, + { + name: "mixed separators", + windowsPath: "/folder\\subfolder/file", + expectedNormPath: "/folder/subfolder/file", + }, + { + name: "already normalized", + windowsPath: "/folder/file", + expectedNormPath: "/folder/file", + }, + { + name: "simple backslash path", + windowsPath: "\\data", + expectedNormPath: "/data", + }, + { + name: "deep nested path", + windowsPath: "\\a\\b\\c\\d", + expectedNormPath: "/a/b/c/d", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Test the actual production function + normalized := util.CleanWindowsPath(tt.windowsPath) + if normalized != tt.expectedNormPath { + t.Errorf("CleanWindowsPath(%q): expected %q, got %q", + tt.windowsPath, tt.expectedNormPath, normalized) + } + }) + } +} + +// TestBreadcrumbPathFormatting validates that breadcrumb paths always +// use forward slashes and maintain proper URL format +func TestBreadcrumbPathFormatting(t *testing.T) { + s := &AdminServer{} + + testPaths := []string{ + "/", + "/folder", + "/folder/subfolder", + "/buckets/mybucket", + "/buckets/mybucket/data", + } + + for _, testPath := range testPaths { + t.Run("breadcrumbs_for_"+testPath, func(t *testing.T) { + breadcrumbs := s.generateBreadcrumbs(testPath) + + // Verify all breadcrumb paths use forward slashes + for i, crumb := range breadcrumbs { + if strings.Contains(crumb.Path, "\\") { + t.Errorf("breadcrumb %d has backslash in path: %q", i, crumb.Path) + } + // Verify paths start with / (except when empty) + if crumb.Path != "" && !strings.HasPrefix(crumb.Path, "/") { + t.Errorf("breadcrumb %d path should start with /: %q", i, crumb.Path) + } + } + }) + } +} + +// TestDirectoryNavigation validates the complete navigation flow +// for various path scenarios +func TestDirectoryNavigation(t *testing.T) { + s := &AdminServer{} + + tests := []struct { + name string + currentPath string + expectedParent string + expectedCrumbs int + }{ + { + name: "navigate from root", + currentPath: "/", + expectedParent: "/", + expectedCrumbs: 1, + }, + { + name: "navigate from single folder", + currentPath: "/documents", + expectedParent: "/", + expectedCrumbs: 2, + }, + { + name: "navigate from nested folder", + currentPath: "/documents/projects/current", + expectedParent: "/documents/projects", + expectedCrumbs: 4, + }, + { + name: "navigate bucket contents", + currentPath: "/buckets/data/files", + expectedParent: "/buckets/data", + expectedCrumbs: 4, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Verify breadcrumbs are generated correctly + breadcrumbs := s.generateBreadcrumbs(tt.currentPath) + if len(breadcrumbs) != tt.expectedCrumbs { + t.Errorf("expected %d breadcrumbs, got %d", tt.expectedCrumbs, len(breadcrumbs)) + } + + // Verify parent path calculation + expectedParent := "/" + if tt.currentPath != "/" { + expectedParent = path.Dir(tt.currentPath) + if expectedParent == "." { + expectedParent = "/" + } + } + if expectedParent != tt.expectedParent { + t.Errorf("expected parent %q, got %q", tt.expectedParent, expectedParent) + } + + // Verify all paths use forward slashes + for i, crumb := range breadcrumbs { + if strings.Contains(crumb.Path, "\\") { + t.Errorf("breadcrumb %d contains backslash: %q", i, crumb.Path) + } + } + }) + } +} diff --git a/weed/admin/dash/middleware.go b/weed/admin/dash/middleware.go index a4cfedfd0..98f3c3d7f 100644 --- a/weed/admin/dash/middleware.go +++ b/weed/admin/dash/middleware.go @@ -2,17 +2,30 @@ package dash import ( "net/http" + "strings" "github.com/gin-contrib/sessions" "github.com/gin-gonic/gin" ) +// setAuthContext sets username and role in context for use in handlers +func setAuthContext(c *gin.Context, username, role interface{}) { + c.Set("username", username) + if role != nil { + c.Set("role", role) + } else { + // Default to admin for backward compatibility + c.Set("role", "admin") + } +} + // RequireAuth checks if user is authenticated func RequireAuth() gin.HandlerFunc { return func(c *gin.Context) { session := sessions.Default(c) authenticated := session.Get("authenticated") username := session.Get("username") + role := session.Get("role") if authenticated != true || username == nil { c.Redirect(http.StatusTemporaryRedirect, "/login") @@ -20,8 +33,8 @@ func RequireAuth() gin.HandlerFunc { return } - // Set username in context for use in handlers - c.Set("username", username) + // Set username and role in context for use in handlers + setAuthContext(c, username, role) c.Next() } } @@ -33,6 +46,7 @@ func RequireAuthAPI() gin.HandlerFunc { session := sessions.Default(c) authenticated := session.Get("authenticated") username := session.Get("username") + role := session.Get("role") if authenticated != true || username == nil { c.JSON(http.StatusUnauthorized, gin.H{ @@ -43,8 +57,37 @@ func RequireAuthAPI() gin.HandlerFunc { return } - // Set username in context for use in handlers - c.Set("username", username) + // Set username and role in context for use in handlers + setAuthContext(c, username, role) + c.Next() + } +} + +// RequireWriteAccess checks if user has admin role (write access) +// Returns JSON error for API endpoints, redirects for HTML endpoints +func RequireWriteAccess() gin.HandlerFunc { + return func(c *gin.Context) { + role, exists := c.Get("role") + if !exists { + role = "admin" // Default for backward compatibility + } + + roleStr, ok := role.(string) + if !ok || roleStr != "admin" { + // Check if this is an API request (path starts with /api) or HTML request + path := c.Request.URL.Path + if strings.HasPrefix(path, "/api") { + c.JSON(http.StatusForbidden, gin.H{ + "error": "Insufficient permissions", + "message": "This operation requires admin access. Read-only users can only view data.", + }) + } else { + c.Redirect(http.StatusSeeOther, "/admin?error=Insufficient permissions") + } + c.Abort() + return + } + c.Next() } } diff --git a/weed/admin/dash/service_account_helpers.go b/weed/admin/dash/service_account_helpers.go new file mode 100644 index 000000000..226f7ec41 --- /dev/null +++ b/weed/admin/dash/service_account_helpers.go @@ -0,0 +1,53 @@ +package dash + +import ( + "fmt" + "strings" + + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" +) + +// identityToServiceAccount converts an IAM identity to a ServiceAccount struct +// This helper reduces code duplication across GetServiceAccounts, GetServiceAccountDetails, +// UpdateServiceAccount, and GetServiceAccountByAccessKey +func identityToServiceAccount(identity *iam_pb.Identity) (*ServiceAccount, error) { + if identity == nil { + return nil, fmt.Errorf("identity cannot be nil") + } + if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) { + return nil, fmt.Errorf("not a service account: %s", identity.GetName()) + } + + parts := strings.SplitN(identity.GetName(), ":", 3) + if len(parts) < 3 { + return nil, fmt.Errorf("invalid service account ID format") + } + + sa := &ServiceAccount{ + ID: identity.GetName(), + ParentUser: parts[1], + Status: StatusActive, + CreateDate: getCreationDate(identity.GetActions()), + Expiration: getExpiration(identity.GetActions()), + } + + // Get description from account display name + if identity.Account != nil { + sa.Description = identity.Account.GetDisplayName() + } + + // Get access key from credentials + if len(identity.Credentials) > 0 { + sa.AccessKeyId = identity.Credentials[0].GetAccessKey() + } + + // Check if disabled + for _, action := range identity.GetActions() { + if action == disabledAction { + sa.Status = StatusInactive + break + } + } + + return sa, nil +} diff --git a/weed/admin/dash/service_account_management.go b/weed/admin/dash/service_account_management.go new file mode 100644 index 000000000..165313888 --- /dev/null +++ b/weed/admin/dash/service_account_management.go @@ -0,0 +1,434 @@ +package dash + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" +) + +var ( + // ErrServiceAccountNotFound is returned when a service account is not found + ErrServiceAccountNotFound = errors.New("service account not found") +) + +const ( + createdAtActionPrefix = "createdAt:" + expirationActionPrefix = "expiresAt:" + disabledAction = "__disabled__" + serviceAccountPrefix = "sa:" + accessKeyPrefix = "ABIA" // Service account access keys use ABIA prefix + + // Status constants + StatusActive = "Active" + StatusInactive = "Inactive" +) + +// Helper functions for managing creation timestamps in actions +func getCreationDate(actions []string) time.Time { + for _, action := range actions { + if strings.HasPrefix(action, createdAtActionPrefix) { + timestampStr := strings.TrimPrefix(action, createdAtActionPrefix) + if timestamp, err := strconv.ParseInt(timestampStr, 10, 64); err == nil { + return time.Unix(timestamp, 0) + } + } + } + return time.Time{} // Return zero time for legacy service accounts without stored creation date +} + +func setCreationDate(actions []string, createDate time.Time) []string { + // Remove any existing createdAt action + filtered := make([]string, 0, len(actions)+1) + for _, action := range actions { + if !strings.HasPrefix(action, createdAtActionPrefix) { + filtered = append(filtered, action) + } + } + // Add new createdAt action + filtered = append(filtered, fmt.Sprintf("%s%d", createdAtActionPrefix, createDate.Unix())) + return filtered +} + +// Helper functions for managing expiration timestamps in actions +func getExpiration(actions []string) time.Time { + for _, action := range actions { + if strings.HasPrefix(action, expirationActionPrefix) { + timestampStr := strings.TrimPrefix(action, expirationActionPrefix) + if timestamp, err := strconv.ParseInt(timestampStr, 10, 64); err == nil { + return time.Unix(timestamp, 0) + } + } + } + return time.Time{} // No expiration set +} + +func setExpiration(actions []string, expiration time.Time) []string { + // Remove any existing expiration action + filtered := make([]string, 0, len(actions)+1) + for _, action := range actions { + if !strings.HasPrefix(action, expirationActionPrefix) { + filtered = append(filtered, action) + } + } + // Add new expiration action if not zero + if !expiration.IsZero() { + filtered = append(filtered, fmt.Sprintf("%s%d", expirationActionPrefix, expiration.Unix())) + } + return filtered +} + +// GetServiceAccounts returns all service accounts, optionally filtered by parent user +// NOTE: Service accounts are stored as special identities with "sa:" prefix +func (s *AdminServer) GetServiceAccounts(ctx context.Context, parentUser string) ([]ServiceAccount, error) { + if s.credentialManager == nil { + return nil, fmt.Errorf("credential manager not available") + } + + // Load the current configuration to find service account identities + config, err := s.credentialManager.LoadConfiguration(ctx) + if err != nil { + return nil, fmt.Errorf("failed to load configuration: %w", err) + } + + var accounts []ServiceAccount + + // Service accounts are stored as identities with "sa:" prefix in their name + // Format: "sa::" + for _, identity := range config.GetIdentities() { + if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) { + continue + } + + parts := strings.SplitN(identity.GetName(), ":", 3) + if len(parts) < 3 { + continue + } + + parent := parts[1] + saId := identity.GetName() + + // Filter by parent user if specified + if parentUser != "" && parent != parentUser { + continue + } + + // Extract description from account display name if available + description := "" + status := StatusActive + if identity.Account != nil { + description = identity.Account.GetDisplayName() + } + + // Get access key from credentials + accessKey := "" + if len(identity.Credentials) > 0 { + accessKey = identity.Credentials[0].GetAccessKey() + // Service accounts use ABIA prefix + if !strings.HasPrefix(accessKey, accessKeyPrefix) { + continue // Not a service account + } + } + + // Check if disabled (stored in actions) + for _, action := range identity.GetActions() { + if action == disabledAction { + status = StatusInactive + break + } + } + + accounts = append(accounts, ServiceAccount{ + ID: saId, + ParentUser: parent, + Description: description, + AccessKeyId: accessKey, + Status: status, + CreateDate: getCreationDate(identity.GetActions()), + Expiration: getExpiration(identity.GetActions()), + }) + } + + return accounts, nil +} + +// GetServiceAccountDetails returns detailed information about a specific service account +func (s *AdminServer) GetServiceAccountDetails(ctx context.Context, id string) (*ServiceAccount, error) { + if s.credentialManager == nil { + return nil, fmt.Errorf("credential manager not available") + } + + // Get the identity + identity, err := s.credentialManager.GetUser(ctx, id) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrServiceAccountNotFound, id) + } + + if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) { + return nil, fmt.Errorf("%w: not a service account: %s", ErrServiceAccountNotFound, id) + } + + parts := strings.SplitN(identity.GetName(), ":", 3) + if len(parts) < 3 { + return nil, fmt.Errorf("invalid service account ID format") + } + + account := &ServiceAccount{ + ID: id, + ParentUser: parts[1], + Status: StatusActive, + CreateDate: getCreationDate(identity.GetActions()), + Expiration: getExpiration(identity.GetActions()), + } + + if identity.Account != nil { + account.Description = identity.Account.GetDisplayName() + } + + if len(identity.Credentials) > 0 { + account.AccessKeyId = identity.Credentials[0].GetAccessKey() + } + + // Check if disabled + for _, action := range identity.GetActions() { + if action == disabledAction { + account.Status = StatusInactive + break + } + } + + return account, nil +} + +// CreateServiceAccount creates a new service account for a parent user +func (s *AdminServer) CreateServiceAccount(ctx context.Context, req CreateServiceAccountRequest) (*ServiceAccount, error) { + if s.credentialManager == nil { + return nil, fmt.Errorf("credential manager not available") + } + + // Validate parent user exists + _, err := s.credentialManager.GetUser(ctx, req.ParentUser) + if err != nil { + return nil, fmt.Errorf("parent user not found: %s", req.ParentUser) + } + + // Generate unique ID and credentials + uuid := generateAccountId() + saId := fmt.Sprintf("sa:%s:%s", req.ParentUser, uuid) + accessKey := accessKeyPrefix + generateAccessKey()[len(accessKeyPrefix):] // Use ABIA prefix for service accounts + secretKey := generateSecretKey() + + // Create the service account as a special identity + now := time.Now() + + // Parse expiration if provided + var expiration time.Time + if req.Expiration != "" { + var err error + expiration, err = time.Parse(time.RFC3339, req.Expiration) + if err != nil { + return nil, fmt.Errorf("invalid expiration format: %w", err) + } + } + + identity := &iam_pb.Identity{ + Name: saId, + Account: &iam_pb.Account{ + Id: uuid, + DisplayName: req.Description, + }, + Credentials: []*iam_pb.Credential{ + { + AccessKey: accessKey, + SecretKey: secretKey, + }, + }, + // Store creation date and expiration in actions + Actions: setExpiration(setCreationDate([]string{}, now), expiration), + } + + // Create the service account + err = s.credentialManager.CreateUser(ctx, identity) + if err != nil { + return nil, fmt.Errorf("failed to create service account: %w", err) + } + + glog.V(1).Infof("Created service account %s for user %s", saId, req.ParentUser) + + return &ServiceAccount{ + ID: saId, + ParentUser: req.ParentUser, + Description: req.Description, + AccessKeyId: accessKey, + SecretAccessKey: secretKey, // Only returned on creation + Status: StatusActive, + CreateDate: now, + Expiration: expiration, + }, nil +} + +// UpdateServiceAccount updates an existing service account +func (s *AdminServer) UpdateServiceAccount(ctx context.Context, id string, req UpdateServiceAccountRequest) (*ServiceAccount, error) { + if s.credentialManager == nil { + return nil, fmt.Errorf("credential manager not available") + } + + // Get existing identity + identity, err := s.credentialManager.GetUser(ctx, id) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrServiceAccountNotFound, id) + } + + if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) { + return nil, fmt.Errorf("%w: not a service account: %s", ErrServiceAccountNotFound, id) + } + + // Update description if provided + if req.Description != "" { + if identity.Account == nil { + identity.Account = &iam_pb.Account{} + } + identity.Account.DisplayName = req.Description + } + + // Update status by adding/removing disabled action + if req.Status != "" { + // Remove existing disabled marker + newActions := make([]string, 0, len(identity.Actions)) + for _, action := range identity.Actions { + if action != disabledAction { + newActions = append(newActions, action) + } + } + // Add disabled action if setting to Inactive + if req.Status == StatusInactive { + newActions = append(newActions, disabledAction) + } + identity.Actions = newActions + } + + // Update expiration if provided + if req.Expiration != "" { + var expiration time.Time + var err error + expiration, err = time.Parse(time.RFC3339, req.Expiration) + if err != nil { + return nil, fmt.Errorf("invalid expiration format: %w", err) + } + identity.Actions = setExpiration(identity.Actions, expiration) + } + + // Update the identity + err = s.credentialManager.UpdateUser(ctx, id, identity) + if err != nil { + return nil, fmt.Errorf("failed to update service account: %w", err) + } + + glog.V(1).Infof("Updated service account %s", id) + + // Build response + parts := strings.SplitN(id, ":", 3) + if len(parts) < 3 { + return nil, fmt.Errorf("invalid service account ID format") + } + + result := &ServiceAccount{ + ID: id, + ParentUser: parts[1], + Description: identity.Account.GetDisplayName(), + Status: StatusActive, + CreateDate: getCreationDate(identity.Actions), + } + + if len(identity.Credentials) > 0 { + result.AccessKeyId = identity.Credentials[0].GetAccessKey() + } + + for _, action := range identity.Actions { + if action == disabledAction { + result.Status = StatusInactive + break + } + } + + return result, nil +} + +// DeleteServiceAccount deletes a service account +func (s *AdminServer) DeleteServiceAccount(ctx context.Context, id string) error { + if s.credentialManager == nil { + return fmt.Errorf("credential manager not available") + } + + // Verify it's a service account + identity, err := s.credentialManager.GetUser(ctx, id) + if err != nil { + return fmt.Errorf("%w: %s", ErrServiceAccountNotFound, id) + } + + if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) { + return fmt.Errorf("%w: not a service account: %s", ErrServiceAccountNotFound, id) + } + + // Delete the identity + err = s.credentialManager.DeleteUser(ctx, id) + if err != nil { + return fmt.Errorf("failed to delete service account: %w", err) + } + + glog.V(1).Infof("Deleted service account %s", id) + return nil +} + +// GetServiceAccountByAccessKey finds a service account by its access key +func (s *AdminServer) GetServiceAccountByAccessKey(ctx context.Context, accessKey string) (*ServiceAccount, error) { + if !strings.HasPrefix(accessKey, accessKeyPrefix) { + return nil, fmt.Errorf("not a service account access key") + } + + if s.credentialManager == nil { + return nil, fmt.Errorf("credential manager not available") + } + + // Find identity by access key + identity, err := s.credentialManager.GetUserByAccessKey(ctx, accessKey) + if err != nil { + return nil, fmt.Errorf("service account not found for access key: %s", accessKey) + } + + if !strings.HasPrefix(identity.GetName(), serviceAccountPrefix) { + return nil, fmt.Errorf("not a service account") + } + + parts := strings.SplitN(identity.GetName(), ":", 3) + if len(parts) < 3 { + return nil, fmt.Errorf("invalid service account ID format") + } + + account := &ServiceAccount{ + ID: identity.GetName(), + ParentUser: parts[1], + AccessKeyId: accessKey, + Status: StatusActive, + CreateDate: getCreationDate(identity.GetActions()), + Expiration: getExpiration(identity.GetActions()), + } + + if identity.Account != nil { + account.Description = identity.Account.GetDisplayName() + } + + for _, action := range identity.GetActions() { + if action == disabledAction { + account.Status = StatusInactive + break + } + } + + return account, nil +} diff --git a/weed/admin/dash/types.go b/weed/admin/dash/types.go index 5c2ac60e8..4881243a5 100644 --- a/weed/admin/dash/types.go +++ b/weed/admin/dash/types.go @@ -78,7 +78,7 @@ type S3Bucket struct { LastModified time.Time `json:"last_modified"` Quota int64 `json:"quota"` // Quota in bytes, 0 means no quota QuotaEnabled bool `json:"quota_enabled"` // Whether quota is enabled - VersioningEnabled bool `json:"versioning_enabled"` // Whether versioning is enabled + VersioningStatus string `json:"versioning_status"` // Versioning status: "" (never enabled), "Enabled", or "Suspended" ObjectLockEnabled bool `json:"object_lock_enabled"` // Whether object lock is enabled ObjectLockMode string `json:"object_lock_mode"` // Object lock mode: "GOVERNANCE" or "COMPLIANCE" ObjectLockDuration int32 `json:"object_lock_duration"` // Default retention duration in days @@ -552,3 +552,49 @@ type CollectionDetailsData struct { SortBy string `json:"sort_by"` SortOrder string `json:"sort_order"` } + +// Service Account management structures +type ServiceAccount struct { + ID string `json:"id"` + ParentUser string `json:"parent_user"` + Description string `json:"description,omitempty"` + AccessKeyId string `json:"access_key_id,omitempty"` + SecretAccessKey string `json:"secret_access_key,omitempty"` // Only returned on creation + Status string `json:"status"` + CreateDate time.Time `json:"create_date"` + Expiration time.Time `json:"expiration,omitempty"` + // ServiceAccountId is used when returning a single ID in some API responses + ServiceAccountId string `json:"service_account_id,omitempty"` + // ServiceAccountIds is used when returning a list of IDs owned by a user + ServiceAccountIds []string `json:"service_account_ids,omitempty"` +} + +type ServiceAccountsData struct { + Username string `json:"username"` + ServiceAccounts []ServiceAccount `json:"service_accounts"` + TotalAccounts int `json:"total_accounts"` + ActiveAccounts int `json:"active_accounts"` + AvailableUsers []string `json:"available_users"` // For parent user dropdown + LastUpdated time.Time `json:"last_updated"` +} + +type CreateServiceAccountRequest struct { + ParentUser string `json:"parent_user"` + Description string `json:"description,omitempty"` + Expiration string `json:"expiration,omitempty"` // RFC3339 format +} + +type UpdateServiceAccountRequest struct { + Status string `json:"status,omitempty"` // Active, Inactive + Description string `json:"description,omitempty"` + Expiration string `json:"expiration,omitempty"` +} + +// STS Configuration display types +type STSConfigData struct { + Enabled bool `json:"enabled"` + Issuer string `json:"issuer,omitempty"` + TokenDuration string `json:"token_duration,omitempty"` + Providers []string `json:"providers,omitempty"` + LastUpdated time.Time `json:"last_updated"` +} diff --git a/weed/admin/dash/user_management.go b/weed/admin/dash/user_management.go index 747c398d7..3a74a675f 100644 --- a/weed/admin/dash/user_management.go +++ b/weed/admin/dash/user_management.go @@ -326,9 +326,10 @@ func generateSecretKey() string { func generateAccountId() string { // Generate 12-digit account ID - b := make([]byte, 8) + b := make([]byte, 4) rand.Read(b) - return fmt.Sprintf("%012d", b[0]<<24|b[1]<<16|b[2]<<8|b[3]) + val := (uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])) + return fmt.Sprintf("%012d", val) } func randomInt(max int) int { diff --git a/weed/admin/handlers/admin_handlers.go b/weed/admin/handlers/admin_handlers.go index 31fa08113..4c493427f 100644 --- a/weed/admin/handlers/admin_handlers.go +++ b/weed/admin/handlers/admin_handlers.go @@ -5,21 +5,24 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/seaweedfs/seaweedfs/weed/admin/dash" "github.com/seaweedfs/seaweedfs/weed/admin/view/app" "github.com/seaweedfs/seaweedfs/weed/admin/view/layout" + "github.com/seaweedfs/seaweedfs/weed/stats" ) // AdminHandlers contains all the HTTP handlers for the admin interface type AdminHandlers struct { - adminServer *dash.AdminServer - authHandlers *AuthHandlers - clusterHandlers *ClusterHandlers - fileBrowserHandlers *FileBrowserHandlers - userHandlers *UserHandlers - policyHandlers *PolicyHandlers - maintenanceHandlers *MaintenanceHandlers - mqHandlers *MessageQueueHandlers + adminServer *dash.AdminServer + authHandlers *AuthHandlers + clusterHandlers *ClusterHandlers + fileBrowserHandlers *FileBrowserHandlers + userHandlers *UserHandlers + policyHandlers *PolicyHandlers + maintenanceHandlers *MaintenanceHandlers + mqHandlers *MessageQueueHandlers + serviceAccountHandlers *ServiceAccountHandlers } // NewAdminHandlers creates a new instance of AdminHandlers @@ -31,23 +34,28 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers { policyHandlers := NewPolicyHandlers(adminServer) maintenanceHandlers := NewMaintenanceHandlers(adminServer) mqHandlers := NewMessageQueueHandlers(adminServer) + serviceAccountHandlers := NewServiceAccountHandlers(adminServer) return &AdminHandlers{ - adminServer: adminServer, - authHandlers: authHandlers, - clusterHandlers: clusterHandlers, - fileBrowserHandlers: fileBrowserHandlers, - userHandlers: userHandlers, - policyHandlers: policyHandlers, - maintenanceHandlers: maintenanceHandlers, - mqHandlers: mqHandlers, + adminServer: adminServer, + authHandlers: authHandlers, + clusterHandlers: clusterHandlers, + fileBrowserHandlers: fileBrowserHandlers, + userHandlers: userHandlers, + policyHandlers: policyHandlers, + maintenanceHandlers: maintenanceHandlers, + mqHandlers: mqHandlers, + serviceAccountHandlers: serviceAccountHandlers, } } // SetupRoutes configures all the routes for the admin interface -func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, password string) { +func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser, adminPassword, readOnlyUser, readOnlyPassword string) { // Health check (no auth required) r.GET("/health", h.HealthCheck) + // Prometheus metrics endpoint (no auth required) + r.GET("/metrics", gin.WrapH(promhttp.HandlerFor(stats.Gather, promhttp.HandlerOpts{}))) + // Favicon route (no auth required) - redirect to static version r.GET("/favicon.ico", func(c *gin.Context) { c.Redirect(http.StatusMovedPermanently, "/static/favicon.ico") @@ -56,7 +64,7 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, if authRequired { // Authentication routes (no auth required) r.GET("/login", h.authHandlers.ShowLogin) - r.POST("/login", h.authHandlers.HandleLogin(username, password)) + r.POST("/login", h.authHandlers.HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword)) r.GET("/logout", h.authHandlers.HandleLogout) // Protected routes group @@ -72,6 +80,7 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, protected.GET("/object-store/buckets/:bucket", h.ShowBucketDetails) protected.GET("/object-store/users", h.userHandlers.ShowObjectStoreUsers) protected.GET("/object-store/policies", h.policyHandlers.ShowPolicies) + protected.GET("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts) // File browser routes protected.GET("/files", h.fileBrowserHandlers.ShowFileBrowser) @@ -80,12 +89,14 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, protected.GET("/cluster/masters", h.clusterHandlers.ShowClusterMasters) protected.GET("/cluster/filers", h.clusterHandlers.ShowClusterFilers) protected.GET("/cluster/volume-servers", h.clusterHandlers.ShowClusterVolumeServers) - protected.GET("/cluster/volumes", h.clusterHandlers.ShowClusterVolumes) - protected.GET("/cluster/volumes/:id/:server", h.clusterHandlers.ShowVolumeDetails) - protected.GET("/cluster/collections", h.clusterHandlers.ShowClusterCollections) - protected.GET("/cluster/collections/:name", h.clusterHandlers.ShowCollectionDetails) - protected.GET("/cluster/ec-shards", h.clusterHandlers.ShowClusterEcShards) - protected.GET("/cluster/ec-volumes/:id", h.clusterHandlers.ShowEcVolumeDetails) + + // Storage management routes + protected.GET("/storage/volumes", h.clusterHandlers.ShowClusterVolumes) + protected.GET("/storage/volumes/:id/:server", h.clusterHandlers.ShowVolumeDetails) + protected.GET("/storage/collections", h.clusterHandlers.ShowClusterCollections) + protected.GET("/storage/collections/:name", h.clusterHandlers.ShowCollectionDetails) + protected.GET("/storage/ec-shards", h.clusterHandlers.ShowClusterEcShards) + protected.GET("/storage/ec-volumes/:id", h.clusterHandlers.ShowEcVolumeDetails) // Message Queue management routes protected.GET("/mq/brokers", h.mqHandlers.ShowBrokers) @@ -96,9 +107,9 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, protected.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue) protected.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers) protected.GET("/maintenance/config", h.maintenanceHandlers.ShowMaintenanceConfig) - protected.POST("/maintenance/config", h.maintenanceHandlers.UpdateMaintenanceConfig) + protected.POST("/maintenance/config", dash.RequireWriteAccess(), h.maintenanceHandlers.UpdateMaintenanceConfig) protected.GET("/maintenance/config/:taskType", h.maintenanceHandlers.ShowTaskConfig) - protected.POST("/maintenance/config/:taskType", h.maintenanceHandlers.UpdateTaskConfig) + protected.POST("/maintenance/config/:taskType", dash.RequireWriteAccess(), h.maintenanceHandlers.UpdateTaskConfig) protected.GET("/maintenance/tasks/:id", h.maintenanceHandlers.ShowTaskDetail) // API routes for AJAX calls @@ -115,45 +126,55 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, s3Api := api.Group("/s3") { s3Api.GET("/buckets", h.adminServer.ListBucketsAPI) - s3Api.POST("/buckets", h.adminServer.CreateBucket) - s3Api.DELETE("/buckets/:bucket", h.adminServer.DeleteBucket) + s3Api.POST("/buckets", dash.RequireWriteAccess(), h.adminServer.CreateBucket) + s3Api.DELETE("/buckets/:bucket", dash.RequireWriteAccess(), h.adminServer.DeleteBucket) s3Api.GET("/buckets/:bucket", h.adminServer.ShowBucketDetails) - s3Api.PUT("/buckets/:bucket/quota", h.adminServer.UpdateBucketQuota) - s3Api.PUT("/buckets/:bucket/owner", h.adminServer.UpdateBucketOwner) + s3Api.PUT("/buckets/:bucket/quota", dash.RequireWriteAccess(), h.adminServer.UpdateBucketQuota) + s3Api.PUT("/buckets/:bucket/owner", dash.RequireWriteAccess(), h.adminServer.UpdateBucketOwner) } // User management API routes usersApi := api.Group("/users") { usersApi.GET("", h.userHandlers.GetUsers) - usersApi.POST("", h.userHandlers.CreateUser) + usersApi.POST("", dash.RequireWriteAccess(), h.userHandlers.CreateUser) usersApi.GET("/:username", h.userHandlers.GetUserDetails) - usersApi.PUT("/:username", h.userHandlers.UpdateUser) - usersApi.DELETE("/:username", h.userHandlers.DeleteUser) - usersApi.POST("/:username/access-keys", h.userHandlers.CreateAccessKey) - usersApi.DELETE("/:username/access-keys/:accessKeyId", h.userHandlers.DeleteAccessKey) + usersApi.PUT("/:username", dash.RequireWriteAccess(), h.userHandlers.UpdateUser) + usersApi.DELETE("/:username", dash.RequireWriteAccess(), h.userHandlers.DeleteUser) + usersApi.POST("/:username/access-keys", dash.RequireWriteAccess(), h.userHandlers.CreateAccessKey) + usersApi.DELETE("/:username/access-keys/:accessKeyId", dash.RequireWriteAccess(), h.userHandlers.DeleteAccessKey) usersApi.GET("/:username/policies", h.userHandlers.GetUserPolicies) - usersApi.PUT("/:username/policies", h.userHandlers.UpdateUserPolicies) + usersApi.PUT("/:username/policies", dash.RequireWriteAccess(), h.userHandlers.UpdateUserPolicies) + } + + // Service Account management API routes + saApi := api.Group("/service-accounts") + { + saApi.GET("", h.serviceAccountHandlers.GetServiceAccounts) + saApi.POST("", dash.RequireWriteAccess(), h.serviceAccountHandlers.CreateServiceAccount) + saApi.GET("/:id", h.serviceAccountHandlers.GetServiceAccountDetails) + saApi.PUT("/:id", dash.RequireWriteAccess(), h.serviceAccountHandlers.UpdateServiceAccount) + saApi.DELETE("/:id", dash.RequireWriteAccess(), h.serviceAccountHandlers.DeleteServiceAccount) } // Object Store Policy management API routes objectStorePoliciesApi := api.Group("/object-store/policies") { objectStorePoliciesApi.GET("", h.policyHandlers.GetPolicies) - objectStorePoliciesApi.POST("", h.policyHandlers.CreatePolicy) + objectStorePoliciesApi.POST("", dash.RequireWriteAccess(), h.policyHandlers.CreatePolicy) objectStorePoliciesApi.GET("/:name", h.policyHandlers.GetPolicy) - objectStorePoliciesApi.PUT("/:name", h.policyHandlers.UpdatePolicy) - objectStorePoliciesApi.DELETE("/:name", h.policyHandlers.DeletePolicy) + objectStorePoliciesApi.PUT("/:name", dash.RequireWriteAccess(), h.policyHandlers.UpdatePolicy) + objectStorePoliciesApi.DELETE("/:name", dash.RequireWriteAccess(), h.policyHandlers.DeletePolicy) objectStorePoliciesApi.POST("/validate", h.policyHandlers.ValidatePolicy) } // File management API routes filesApi := api.Group("/files") { - filesApi.DELETE("/delete", h.fileBrowserHandlers.DeleteFile) - filesApi.DELETE("/delete-multiple", h.fileBrowserHandlers.DeleteMultipleFiles) - filesApi.POST("/create-folder", h.fileBrowserHandlers.CreateFolder) - filesApi.POST("/upload", h.fileBrowserHandlers.UploadFile) + filesApi.DELETE("/delete", dash.RequireWriteAccess(), h.fileBrowserHandlers.DeleteFile) + filesApi.DELETE("/delete-multiple", dash.RequireWriteAccess(), h.fileBrowserHandlers.DeleteMultipleFiles) + filesApi.POST("/create-folder", dash.RequireWriteAccess(), h.fileBrowserHandlers.CreateFolder) + filesApi.POST("/upload", dash.RequireWriteAccess(), h.fileBrowserHandlers.UploadFile) filesApi.GET("/download", h.fileBrowserHandlers.DownloadFile) filesApi.GET("/view", h.fileBrowserHandlers.ViewFile) filesApi.GET("/properties", h.fileBrowserHandlers.GetFileProperties) @@ -162,32 +183,32 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, // Volume management API routes volumeApi := api.Group("/volumes") { - volumeApi.POST("/:id/:server/vacuum", h.clusterHandlers.VacuumVolume) + volumeApi.POST("/:id/:server/vacuum", dash.RequireWriteAccess(), h.clusterHandlers.VacuumVolume) } // Maintenance API routes maintenanceApi := api.Group("/maintenance") { - maintenanceApi.POST("/scan", h.adminServer.TriggerMaintenanceScan) + maintenanceApi.POST("/scan", dash.RequireWriteAccess(), h.adminServer.TriggerMaintenanceScan) maintenanceApi.GET("/tasks", h.adminServer.GetMaintenanceTasks) maintenanceApi.GET("/tasks/:id", h.adminServer.GetMaintenanceTask) maintenanceApi.GET("/tasks/:id/detail", h.adminServer.GetMaintenanceTaskDetailAPI) - maintenanceApi.POST("/tasks/:id/cancel", h.adminServer.CancelMaintenanceTask) + maintenanceApi.POST("/tasks/:id/cancel", dash.RequireWriteAccess(), h.adminServer.CancelMaintenanceTask) maintenanceApi.GET("/workers", h.adminServer.GetMaintenanceWorkersAPI) maintenanceApi.GET("/workers/:id", h.adminServer.GetMaintenanceWorker) maintenanceApi.GET("/workers/:id/logs", h.adminServer.GetWorkerLogs) maintenanceApi.GET("/stats", h.adminServer.GetMaintenanceStats) maintenanceApi.GET("/config", h.adminServer.GetMaintenanceConfigAPI) - maintenanceApi.PUT("/config", h.adminServer.UpdateMaintenanceConfigAPI) + maintenanceApi.PUT("/config", dash.RequireWriteAccess(), h.adminServer.UpdateMaintenanceConfigAPI) } // Message Queue API routes mqApi := api.Group("/mq") { mqApi.GET("/topics/:namespace/:topic", h.mqHandlers.GetTopicDetailsAPI) - mqApi.POST("/topics/create", h.mqHandlers.CreateTopicAPI) - mqApi.POST("/topics/retention/update", h.mqHandlers.UpdateTopicRetentionAPI) - mqApi.POST("/retention/purge", h.adminServer.TriggerTopicRetentionPurgeAPI) + mqApi.POST("/topics/create", dash.RequireWriteAccess(), h.mqHandlers.CreateTopicAPI) + mqApi.POST("/topics/retention/update", dash.RequireWriteAccess(), h.mqHandlers.UpdateTopicRetentionAPI) + mqApi.POST("/retention/purge", dash.RequireWriteAccess(), h.adminServer.TriggerTopicRetentionPurgeAPI) } } } else { @@ -200,6 +221,7 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, r.GET("/object-store/buckets/:bucket", h.ShowBucketDetails) r.GET("/object-store/users", h.userHandlers.ShowObjectStoreUsers) r.GET("/object-store/policies", h.policyHandlers.ShowPolicies) + r.GET("/object-store/service-accounts", h.serviceAccountHandlers.ShowServiceAccounts) // File browser routes r.GET("/files", h.fileBrowserHandlers.ShowFileBrowser) @@ -208,12 +230,14 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, r.GET("/cluster/masters", h.clusterHandlers.ShowClusterMasters) r.GET("/cluster/filers", h.clusterHandlers.ShowClusterFilers) r.GET("/cluster/volume-servers", h.clusterHandlers.ShowClusterVolumeServers) - r.GET("/cluster/volumes", h.clusterHandlers.ShowClusterVolumes) - r.GET("/cluster/volumes/:id/:server", h.clusterHandlers.ShowVolumeDetails) - r.GET("/cluster/collections", h.clusterHandlers.ShowClusterCollections) - r.GET("/cluster/collections/:name", h.clusterHandlers.ShowCollectionDetails) - r.GET("/cluster/ec-shards", h.clusterHandlers.ShowClusterEcShards) - r.GET("/cluster/ec-volumes/:id", h.clusterHandlers.ShowEcVolumeDetails) + + // Storage management routes + r.GET("/storage/volumes", h.clusterHandlers.ShowClusterVolumes) + r.GET("/storage/volumes/:id/:server", h.clusterHandlers.ShowVolumeDetails) + r.GET("/storage/collections", h.clusterHandlers.ShowClusterCollections) + r.GET("/storage/collections/:name", h.clusterHandlers.ShowCollectionDetails) + r.GET("/storage/ec-shards", h.clusterHandlers.ShowClusterEcShards) + r.GET("/storage/ec-volumes/:id", h.clusterHandlers.ShowEcVolumeDetails) // Message Queue management routes r.GET("/mq/brokers", h.mqHandlers.ShowBrokers) @@ -263,6 +287,16 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, username, usersApi.PUT("/:username/policies", h.userHandlers.UpdateUserPolicies) } + // Service Account management API routes + saApi := api.Group("/service-accounts") + { + saApi.GET("", h.serviceAccountHandlers.GetServiceAccounts) + saApi.POST("", h.serviceAccountHandlers.CreateServiceAccount) + saApi.GET("/:id", h.serviceAccountHandlers.GetServiceAccountDetails) + saApi.PUT("/:id", h.serviceAccountHandlers.UpdateServiceAccount) + saApi.DELETE("/:id", h.serviceAccountHandlers.DeleteServiceAccount) + } + // Object Store Policy management API routes objectStorePoliciesApi := api.Group("/object-store/policies") { diff --git a/weed/admin/handlers/auth_handlers.go b/weed/admin/handlers/auth_handlers.go index 07596b8e4..ff6f6250e 100644 --- a/weed/admin/handlers/auth_handlers.go +++ b/weed/admin/handlers/auth_handlers.go @@ -3,6 +3,7 @@ package handlers import ( "net/http" + "github.com/gin-contrib/sessions" "github.com/gin-gonic/gin" "github.com/seaweedfs/seaweedfs/weed/admin/dash" "github.com/seaweedfs/seaweedfs/weed/admin/view/layout" @@ -22,6 +23,14 @@ func NewAuthHandlers(adminServer *dash.AdminServer) *AuthHandlers { // ShowLogin displays the login page func (a *AuthHandlers) ShowLogin(c *gin.Context) { + session := sessions.Default(c) + + // If already authenticated, redirect to admin + if session.Get("authenticated") == true { + c.Redirect(http.StatusSeeOther, "/admin") + return + } + errorMessage := c.Query("error") // Render login template @@ -35,8 +44,8 @@ func (a *AuthHandlers) ShowLogin(c *gin.Context) { } // HandleLogin handles login form submission -func (a *AuthHandlers) HandleLogin(username, password string) gin.HandlerFunc { - return a.adminServer.HandleLogin(username, password) +func (a *AuthHandlers) HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword string) gin.HandlerFunc { + return a.adminServer.HandleLogin(adminUser, adminPassword, readOnlyUser, readOnlyPassword) } // HandleLogout handles user logout diff --git a/weed/admin/handlers/file_browser_handlers.go b/weed/admin/handlers/file_browser_handlers.go index eeb8e2d85..b7c44a69d 100644 --- a/weed/admin/handlers/file_browser_handlers.go +++ b/weed/admin/handlers/file_browser_handlers.go @@ -60,9 +60,22 @@ func (h *FileBrowserHandlers) newClientWithTimeout(timeout time.Duration) http.C func (h *FileBrowserHandlers) ShowFileBrowser(c *gin.Context) { // Get path from query parameter, default to root path := c.DefaultQuery("path", "/") + // Normalize Windows-style paths for consistency + path = util.CleanWindowsPath(path) - // Get file browser data - browserData, err := h.adminServer.GetFileBrowser(path) + // Get pagination parameters + lastFileName := c.DefaultQuery("lastFileName", "") + + pageSize, err := strconv.Atoi(c.DefaultQuery("limit", "20")) + if err != nil || pageSize < 1 { + pageSize = 20 + } + if pageSize > 200 { + pageSize = 200 + } + + // Get file browser data with cursor-based pagination + browserData, err := h.adminServer.GetFileBrowser(path, lastFileName, pageSize) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get file browser data: " + err.Error()}) return diff --git a/weed/admin/handlers/service_account_handlers.go b/weed/admin/handlers/service_account_handlers.go new file mode 100644 index 000000000..9c8c85f9a --- /dev/null +++ b/weed/admin/handlers/service_account_handlers.go @@ -0,0 +1,213 @@ +package handlers + +import ( + "bytes" + "errors" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/seaweedfs/seaweedfs/weed/admin/dash" + "github.com/seaweedfs/seaweedfs/weed/admin/view/app" + "github.com/seaweedfs/seaweedfs/weed/admin/view/layout" + "github.com/seaweedfs/seaweedfs/weed/glog" +) + +// ServiceAccountHandlers contains HTTP handlers for service account management +type ServiceAccountHandlers struct { + adminServer *dash.AdminServer +} + +// NewServiceAccountHandlers creates a new instance of ServiceAccountHandlers +func NewServiceAccountHandlers(adminServer *dash.AdminServer) *ServiceAccountHandlers { + return &ServiceAccountHandlers{ + adminServer: adminServer, + } +} + +// ShowServiceAccounts renders the service accounts management page +func (h *ServiceAccountHandlers) ShowServiceAccounts(c *gin.Context) { + data := h.getServiceAccountsData(c) + + // Render to buffer first to avoid partial writes on error + var buf bytes.Buffer + component := app.ServiceAccounts(data) + layoutComponent := layout.Layout(c, component) + err := layoutComponent.Render(c.Request.Context(), &buf) + if err != nil { + glog.Errorf("Failed to render service accounts template: %v", err) + c.AbortWithStatus(http.StatusInternalServerError) + return + } + + // Only write to response if render succeeded + c.Header("Content-Type", "text/html") + c.Writer.Write(buf.Bytes()) +} + +// GetServiceAccounts returns the list of service accounts as JSON +func (h *ServiceAccountHandlers) GetServiceAccounts(c *gin.Context) { + parentUser := c.Query("parent_user") + + accounts, err := h.adminServer.GetServiceAccounts(c.Request.Context(), parentUser) + if err != nil { + glog.Errorf("Failed to get service accounts: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get service accounts"}) + return + } + c.JSON(http.StatusOK, gin.H{"service_accounts": accounts}) +} + +// CreateServiceAccount handles service account creation +func (h *ServiceAccountHandlers) CreateServiceAccount(c *gin.Context) { + var req dash.CreateServiceAccountRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()}) + return + } + + if req.ParentUser == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "ParentUser is required"}) + return + } + + sa, err := h.adminServer.CreateServiceAccount(c.Request.Context(), req) + if err != nil { + glog.Errorf("Failed to create service account for user %s: %v", req.ParentUser, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create service account"}) + return + } + + c.JSON(http.StatusCreated, gin.H{ + "message": "Service account created successfully", + "service_account": sa, + }) +} + +// GetServiceAccountDetails returns detailed information about a service account +func (h *ServiceAccountHandlers) GetServiceAccountDetails(c *gin.Context) { + id := c.Param("id") + if id == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "Service account ID is required"}) + return + } + + sa, err := h.adminServer.GetServiceAccountDetails(c.Request.Context(), id) + if err != nil { + // Distinguish not-found errors from internal errors + if errors.Is(err, dash.ErrServiceAccountNotFound) { + c.JSON(http.StatusNotFound, gin.H{"error": "Service account not found: " + err.Error()}) + } else { + glog.Errorf("Failed to get service account details for %s: %v", id, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get service account details"}) + } + return + } + + c.JSON(http.StatusOK, sa) +} + +// UpdateServiceAccount handles service account updates +func (h *ServiceAccountHandlers) UpdateServiceAccount(c *gin.Context) { + id := c.Param("id") + if id == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "Service account ID is required"}) + return + } + + var req dash.UpdateServiceAccountRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid request: " + err.Error()}) + return + } + + sa, err := h.adminServer.UpdateServiceAccount(c.Request.Context(), id, req) + if err != nil { + // Distinguish not-found errors from internal errors + if errors.Is(err, dash.ErrServiceAccountNotFound) { + c.JSON(http.StatusNotFound, gin.H{"error": "Service account not found"}) + } else { + glog.Errorf("Failed to update service account %s: %v", id, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to update service account"}) + } + return + } + + c.JSON(http.StatusOK, gin.H{ + "message": "Service account updated successfully", + "service_account": sa, + }) +} + +// DeleteServiceAccount handles service account deletion +func (h *ServiceAccountHandlers) DeleteServiceAccount(c *gin.Context) { + id := c.Param("id") + if id == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "Service account ID is required"}) + return + } + + err := h.adminServer.DeleteServiceAccount(c.Request.Context(), id) + if err != nil { + // Distinguish not-found errors from internal errors + if errors.Is(err, dash.ErrServiceAccountNotFound) { + c.JSON(http.StatusNotFound, gin.H{"error": "Service account not found"}) + } else { + glog.Errorf("Failed to delete service account %s: %v", id, err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete service account"}) + } + return + } + + c.JSON(http.StatusOK, gin.H{ + "message": "Service account deleted successfully", + }) +} + +// getServiceAccountsData retrieves service accounts data for the template +func (h *ServiceAccountHandlers) getServiceAccountsData(c *gin.Context) dash.ServiceAccountsData { + username := c.GetString("username") + if username == "" { + username = "admin" + } + + // Get all service accounts + accounts, err := h.adminServer.GetServiceAccounts(c.Request.Context(), "") + if err != nil { + glog.Errorf("Failed to get service accounts: %v", err) + return dash.ServiceAccountsData{ + Username: username, + ServiceAccounts: []dash.ServiceAccount{}, + TotalAccounts: 0, + LastUpdated: time.Now(), + } + } + + // Count active accounts + activeCount := 0 + for _, sa := range accounts { + if sa.Status == dash.StatusActive { + activeCount++ + } + } + + // Get available users for dropdown + var availableUsers []string + users, err := h.adminServer.GetObjectStoreUsers(c.Request.Context()) + if err != nil { + glog.Errorf("Failed to get users for dropdown: %v", err) + } else { + for _, user := range users { + availableUsers = append(availableUsers, user.Username) + } + } + + return dash.ServiceAccountsData{ + Username: username, + ServiceAccounts: accounts, + TotalAccounts: len(accounts), + ActiveAccounts: activeCount, + AvailableUsers: availableUsers, + LastUpdated: time.Now(), + } +} diff --git a/weed/admin/handlers/user_handlers.go b/weed/admin/handlers/user_handlers.go index 9f36848c0..ed280a0ad 100644 --- a/weed/admin/handlers/user_handlers.go +++ b/weed/admin/handlers/user_handlers.go @@ -41,7 +41,7 @@ func (h *UserHandlers) ShowObjectStoreUsers(c *gin.Context) { // GetUsers returns the list of users as JSON func (h *UserHandlers) GetUsers(c *gin.Context) { - users, err := h.adminServer.GetObjectStoreUsers() + users, err := h.adminServer.GetObjectStoreUsers(c.Request.Context()) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to get users: " + err.Error()}) return @@ -234,7 +234,7 @@ func (h *UserHandlers) getObjectStoreUsersData(c *gin.Context) dash.ObjectStoreU } // Get object store users - users, err := h.adminServer.GetObjectStoreUsers() + users, err := h.adminServer.GetObjectStoreUsers(c.Request.Context()) if err != nil { glog.Errorf("Failed to get object store users: %v", err) // Return empty data on error diff --git a/weed/admin/maintenance/maintenance_scanner.go b/weed/admin/maintenance/maintenance_scanner.go index 6f3b46be2..ddbf44f55 100644 --- a/weed/admin/maintenance/maintenance_scanner.go +++ b/weed/admin/maintenance/maintenance_scanner.go @@ -115,6 +115,7 @@ func (ms *MaintenanceScanner) getVolumeHealthMetrics() ([]*VolumeHealthMetrics, metric := &VolumeHealthMetrics{ VolumeID: volInfo.Id, Server: node.Id, + ServerAddress: node.Address, DiskType: diskType, // Track which disk this volume is on DiskId: volInfo.DiskId, // Use disk ID from volume info DataCenter: dc.Id, // Data center from current loop @@ -207,6 +208,7 @@ func (ms *MaintenanceScanner) convertToTaskMetrics(metrics []*VolumeHealthMetric simplified = append(simplified, &types.VolumeHealthMetrics{ VolumeID: metric.VolumeID, Server: metric.Server, + ServerAddress: metric.ServerAddress, DiskType: metric.DiskType, DiskId: metric.DiskId, DataCenter: metric.DataCenter, diff --git a/weed/admin/maintenance/maintenance_types.go b/weed/admin/maintenance/maintenance_types.go index fe5d5fa55..7b7e83818 100644 --- a/weed/admin/maintenance/maintenance_types.go +++ b/weed/admin/maintenance/maintenance_types.go @@ -362,6 +362,7 @@ type TaskDetectionResult struct { type VolumeHealthMetrics struct { VolumeID uint32 `json:"volume_id"` Server string `json:"server"` + ServerAddress string `json:"server_address"` DiskType string `json:"disk_type"` // Disk type (e.g., "hdd", "ssd") or disk path (e.g., "/data1") DiskId uint32 `json:"disk_id"` // ID of the disk in Store.Locations array DataCenter string `json:"data_center"` // Data center of the server diff --git a/weed/admin/maintenance/pending_operations_test.go b/weed/admin/maintenance/pending_operations_test.go index 64bb591fb..75d511e8f 100644 --- a/weed/admin/maintenance/pending_operations_test.go +++ b/weed/admin/maintenance/pending_operations_test.go @@ -110,10 +110,10 @@ func TestPendingOperations_VolumeFiltering(t *testing.T) { // Create volume metrics metrics := []*types.VolumeHealthMetrics{ - {VolumeID: 100, Server: "node1"}, - {VolumeID: 101, Server: "node2"}, - {VolumeID: 102, Server: "node3"}, - {VolumeID: 103, Server: "node1"}, + {VolumeID: 100, Server: "node1", ServerAddress: "192.168.1.1:8080"}, + {VolumeID: 101, Server: "node2", ServerAddress: "192.168.1.2:8080"}, + {VolumeID: 102, Server: "node3", ServerAddress: "192.168.1.3:8080"}, + {VolumeID: 103, Server: "node1", ServerAddress: "192.168.1.1:8080"}, } // Add pending operations on volumes 101 and 103 diff --git a/weed/admin/topology/structs.go b/weed/admin/topology/structs.go index 103ee5abe..06903352e 100644 --- a/weed/admin/topology/structs.go +++ b/weed/admin/topology/structs.go @@ -97,6 +97,7 @@ type ActiveTopology struct { // DestinationPlan represents a planned destination for a volume/shard operation type DestinationPlan struct { TargetNode string `json:"target_node"` + TargetAddress string `json:"target_address"` TargetDisk uint32 `json:"target_disk"` TargetRack string `json:"target_rack"` TargetDC string `json:"target_dc"` diff --git a/weed/admin/view/app/cluster_collections.templ b/weed/admin/view/app/cluster_collections.templ index d4765ea86..52482927f 100644 --- a/weed/admin/view/app/cluster_collections.templ +++ b/weed/admin/view/app/cluster_collections.templ @@ -149,12 +149,12 @@ templ ClusterCollections(data dash.ClusterCollectionsData) { for _, collection := range data.Collections { - + {collection.Name} - +
if collection.VolumeCount > 0 { @@ -166,7 +166,7 @@ templ ClusterCollections(data dash.ClusterCollectionsData) { - +
if collection.EcVolumeCount > 0 { @@ -330,10 +330,10 @@ templ ClusterCollections(data dash.ClusterCollectionsData) { '
' + '
Quick Actions
' + '
' + - '' + + '' + 'View Volumes' + '' + - '' + + '' + 'View EC Volumes' + '' + '' + diff --git a/weed/admin/view/app/cluster_collections_templ.go b/weed/admin/view/app/cluster_collections_templ.go index e3630d7a6..1e0234cbd 100644 --- a/weed/admin/view/app/cluster_collections_templ.go +++ b/weed/admin/view/app/cluster_collections_templ.go @@ -114,7 +114,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component { return templ_7745c5c3_Err } var templ_7745c5c3_Var7 templ.SafeURL - templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/cluster/collections/%s", collection.Name))) + templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/storage/collections/%s", collection.Name))) if templ_7745c5c3_Err != nil { return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 152, Col: 123} } @@ -140,7 +140,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component { return templ_7745c5c3_Err } var templ_7745c5c3_Var9 templ.SafeURL - templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/cluster/volumes?collection=%s", collection.Name))) + templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/storage/volumes?collection=%s", collection.Name))) if templ_7745c5c3_Err != nil { return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 157, Col: 130} } @@ -173,7 +173,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component { return templ_7745c5c3_Err } var templ_7745c5c3_Var11 templ.SafeURL - templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/cluster/ec-shards?collection=%s", collection.Name))) + templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/storage/ec-shards?collection=%s", collection.Name))) if templ_7745c5c3_Err != nil { return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_collections.templ`, Line: 169, Col: 132} } @@ -403,7 +403,7 @@ func ClusterCollections(data dash.ClusterCollectionsData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "
") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 36, "
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/admin/view/app/cluster_ec_shards.templ b/weed/admin/view/app/cluster_ec_shards.templ index a3e8fc0ec..19f6fd2d6 100644 --- a/weed/admin/view/app/cluster_ec_shards.templ +++ b/weed/admin/view/app/cluster_ec_shards.templ @@ -22,7 +22,7 @@ templ ClusterEcShards(data dash.ClusterEcShardsData) { Collection: {data.FilterCollection} } - + Clear Filter @@ -205,11 +205,11 @@ templ ClusterEcShards(data dash.ClusterEcShardsData) { if data.ShowCollectionColumn { if shard.Collection != "" { - + {shard.Collection} } else { - + default } @@ -366,7 +366,7 @@ templ ClusterEcShards(data dash.ClusterEcShardsData) { } function exportEcShards() { - const url = new URL('/api/cluster/ec-shards/export', window.location.origin); + const url = new URL('/api/storage/ec-shards/export', window.location.origin); const params = new URLSearchParams(window.location.search); params.forEach((value, key) => { url.searchParams.set(key, value); @@ -380,7 +380,7 @@ templ ClusterEcShards(data dash.ClusterEcShardsData) { const volumeId = button.getAttribute('data-volume-id'); // Navigate to the EC volume details page - window.location.href = `/cluster/ec-volumes/${volumeId}`; + window.location.href = `/storage/ec-volumes/${volumeId}`; } function repairVolume(event) { @@ -388,7 +388,7 @@ templ ClusterEcShards(data dash.ClusterEcShardsData) { const button = event.target.closest('button'); const volumeId = button.getAttribute('data-volume-id'); if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) { - fetch(`/api/cluster/volumes/${volumeId}/repair`, { + fetch(`/api/storage/volumes/${volumeId}/repair`, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/weed/admin/view/app/cluster_ec_shards_templ.go b/weed/admin/view/app/cluster_ec_shards_templ.go index f995e5ef4..b7c169d1e 100644 --- a/weed/admin/view/app/cluster_ec_shards_templ.go +++ b/weed/admin/view/app/cluster_ec_shards_templ.go @@ -67,7 +67,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "Clear Filter") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "Clear Filter") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -328,7 +328,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component { return templ_7745c5c3_Err } if shard.Collection != "" { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 48, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -346,7 +346,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component { return templ_7745c5c3_Err } } else { - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "default") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 50, "default") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -663,7 +663,7 @@ func ClusterEcShards(data dash.ClusterEcShardsData) templ.Component { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 89, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 89, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/admin/view/app/cluster_ec_volumes.templ b/weed/admin/view/app/cluster_ec_volumes.templ index f5210082e..6e94443ae 100644 --- a/weed/admin/view/app/cluster_ec_volumes.templ +++ b/weed/admin/view/app/cluster_ec_volumes.templ @@ -25,7 +25,7 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) { Collection: {data.Collection} } - + Clear Filter @@ -201,11 +201,11 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) { if data.ShowCollectionColumn { if volume.Collection != "" { - + {volume.Collection} } else { - + default } @@ -373,13 +373,13 @@ templ ClusterEcVolumes(data dash.ClusterEcVolumesData) { function showVolumeDetails(event) { const volumeId = event.target.closest('button').getAttribute('data-volume-id'); - window.location.href = `/cluster/ec-volumes/${volumeId}`; + window.location.href = `/storage/ec-volumes/${volumeId}`; } function repairVolume(event) { const volumeId = event.target.closest('button').getAttribute('data-volume-id'); if (confirm(`Are you sure you want to repair missing shards for volume ${volumeId}?`)) { - fetch(`/api/cluster/ec-volumes/${volumeId}/repair`, { + fetch(`/api/storage/ec-volumes/${volumeId}/repair`, { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/weed/admin/view/app/cluster_ec_volumes_templ.go b/weed/admin/view/app/cluster_ec_volumes_templ.go index ddfd0795a..9103607a8 100644 --- a/weed/admin/view/app/cluster_ec_volumes_templ.go +++ b/weed/admin/view/app/cluster_ec_volumes_templ.go @@ -70,7 +70,7 @@ func ClusterEcVolumes(data dash.ClusterEcVolumesData) templ.Component { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "Clear Filter") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "Clear Filter") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -383,7 +383,7 @@ func ClusterEcVolumes(data dash.ClusterEcVolumesData) templ.Component { return templ_7745c5c3_Err } var templ_7745c5c3_Var13 templ.SafeURL - templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinURLErrs(templ.URL(fmt.Sprintf("/cluster/ec-shards?collection=%s", volume.Collection))) + templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinURLErrs(templ.URL(fmt.Sprintf("/storage/ec-shards?collection=%s", volume.Collection))) if templ_7745c5c3_Err != nil { return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_volumes.templ`, Line: 204, Col: 123} } @@ -414,7 +414,7 @@ func ClusterEcVolumes(data dash.ClusterEcVolumesData) templ.Component { return templ_7745c5c3_Err } var templ_7745c5c3_Var15 templ.SafeURL - templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinURLErrs(templ.URL("/cluster/ec-shards?collection=default")) + templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinURLErrs(templ.URL("/storage/ec-shards?collection=default")) if templ_7745c5c3_Err != nil { return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_ec_volumes.templ`, Line: 208, Col: 96} } @@ -757,7 +757,7 @@ func ClusterEcVolumes(data dash.ClusterEcVolumesData) templ.Component { return templ_7745c5c3_Err } } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 96, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/admin/view/app/cluster_volume_servers.templ b/weed/admin/view/app/cluster_volume_servers.templ index b6de9ad12..65b7f08a1 100644 --- a/weed/admin/view/app/cluster_volume_servers.templ +++ b/weed/admin/view/app/cluster_volume_servers.templ @@ -337,7 +337,7 @@ templ ClusterVolumeServers(data dash.ClusterVolumeServersData) { '' + 'Open Volume Server UI' + '' + - '' + + '' + 'View Volumes' + '' + '' + diff --git a/weed/admin/view/app/cluster_volume_servers_templ.go b/weed/admin/view/app/cluster_volume_servers_templ.go index c7a4ec80b..3a47df7d9 100644 --- a/weed/admin/view/app/cluster_volume_servers_templ.go +++ b/weed/admin/view/app/cluster_volume_servers_templ.go @@ -656,7 +656,7 @@ func ClusterVolumeServers(data dash.ClusterVolumeServersData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/admin/view/app/cluster_volumes.templ b/weed/admin/view/app/cluster_volumes.templ index 8f0b59698..c3b2591da 100644 --- a/weed/admin/view/app/cluster_volumes.templ +++ b/weed/admin/view/app/cluster_volumes.templ @@ -17,7 +17,7 @@ templ ClusterVolumes(data dash.ClusterVolumesData) { Collection: {data.FilterCollection} - + Clear Filter @@ -338,11 +338,11 @@ templ ClusterVolumes(data dash.ClusterVolumesData) { if data.ShowCollectionColumn { if volume.Collection == "" { - + default } else { - + {volume.Collection} } @@ -597,7 +597,7 @@ templ ClusterVolumes(data dash.ClusterVolumesData) { const serverCell = row.querySelector('td:nth-child(2) a'); const server = serverCell ? serverCell.textContent.trim() : 'unknown'; - window.location.href = `/cluster/volumes/${volumeId}/${encodeURIComponent(server)}`; + window.location.href = `/storage/volumes/${volumeId}/${encodeURIComponent(server)}`; } function performVacuum(volumeId, server, button) { diff --git a/weed/admin/view/app/cluster_volumes_templ.go b/weed/admin/view/app/cluster_volumes_templ.go index d96a991ce..117ae8585 100644 --- a/weed/admin/view/app/cluster_volumes_templ.go +++ b/weed/admin/view/app/cluster_volumes_templ.go @@ -53,7 +53,7 @@ func ClusterVolumes(data dash.ClusterVolumesData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, " Clear Filter") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, " Clear Filter") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -557,7 +557,7 @@ func ClusterVolumes(data dash.ClusterVolumesData) templ.Component { return templ_7745c5c3_Err } var templ_7745c5c3_Var21 templ.SafeURL - templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL("/cluster/volumes?collection=default")) + templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL("/storage/volumes?collection=default")) if templ_7745c5c3_Err != nil { return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_volumes.templ`, Line: 341, Col: 113} } @@ -575,7 +575,7 @@ func ClusterVolumes(data dash.ClusterVolumesData) templ.Component { return templ_7745c5c3_Err } var templ_7745c5c3_Var22 templ.SafeURL - templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/cluster/volumes?collection=%s", volume.Collection))) + templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(fmt.Sprintf("/storage/volumes?collection=%s", volume.Collection))) if templ_7745c5c3_Err != nil { return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/app/cluster_volumes.templ`, Line: 345, Col: 140} } @@ -1035,7 +1035,7 @@ func ClusterVolumes(data dash.ClusterVolumesData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 111, "") + templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 111, "") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } diff --git a/weed/admin/view/app/collection_details.templ b/weed/admin/view/app/collection_details.templ index b5c86ba18..296839b93 100644 --- a/weed/admin/view/app/collection_details.templ +++ b/weed/admin/view/app/collection_details.templ @@ -15,7 +15,7 @@ templ CollectionDetails(data dash.CollectionDetailsData) { @@ -360,13 +360,13 @@ templ CollectionDetails(data dash.CollectionDetailsData) { function showVolumeDetails(event) { const volumeId = event.target.closest('button').getAttribute('data-volume-id'); const server = event.target.closest('button').getAttribute('data-server'); - window.location.href = `/cluster/volumes/${volumeId}/${server}`; + window.location.href = `/storage/volumes/${volumeId}/${server}`; } // EC Volume details function showEcVolumeDetails(event) { const volumeId = event.target.closest('button').getAttribute('data-volume-id'); - window.location.href = `/cluster/ec-volumes/${volumeId}`; + window.location.href = `/storage/ec-volumes/${volumeId}`; } // Repair EC Volume diff --git a/weed/admin/view/app/collection_details_templ.go b/weed/admin/view/app/collection_details_templ.go index a0e781637..f2ff0ab13 100644 --- a/weed/admin/view/app/collection_details_templ.go +++ b/weed/admin/view/app/collection_details_templ.go @@ -48,7 +48,7 @@ func CollectionDetails(data dash.CollectionDetailsData) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "