Compare commits

..
239 changed files with 2062 additions and 19927 deletions
@@ -60,16 +60,16 @@ jobs:
build_args: TAGS=5BytesOffset
tag_suffix: _large_disk
# Full tags - multi-arch
# Full tags - amd64 only
- variant: full
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _full
# Large disk + full tags - multi-arch
# Large disk + full tags - amd64 only
- variant: large_disk_full
platforms: linux/amd64,linux/arm64
platforms: linux/amd64
dockerfile: ./docker/Dockerfile.go_build
build_args: TAGS=5BytesOffset,elastic,gocdk,rclone,sqlite,tarantool,tikv,ydb
tag_suffix: _large_disk_full
+1 -1
View File
@@ -135,7 +135,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: output-logs
path: docker/output.log
+1 -1
View File
@@ -52,7 +52,7 @@ jobs:
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ec-integration-test-logs
path: |
+1 -1
View File
@@ -42,7 +42,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ec-test-logs
path: test/erasure_coding/admin_dockertest/tmp/logs/
+1 -1
View File
@@ -183,7 +183,7 @@ jobs:
- name: Upload Test Artifacts
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: fuse-integration-test-results
path: |
+1 -1
View File
@@ -49,7 +49,7 @@ jobs:
- name: Upload Test Reports
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: test-reports-java-${{ matrix.java }}
path: |
@@ -70,7 +70,7 @@ jobs:
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: metadata-subscribe-test-logs
path: |
+1 -1
View File
@@ -62,7 +62,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: postgres-logs
path: test/postgres/postgres-output.log
@@ -57,7 +57,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: integration-test-logs
path: test/s3/normal/*.log
+1 -1
View File
@@ -77,7 +77,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-filer-group-test-logs
path: test/s3/filer_group/weed-test*.log
+9 -9
View File
@@ -76,7 +76,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-test-logs-${{ matrix.test-type }}
path: test/s3/versioning/weed-test*.log
@@ -124,7 +124,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-compatibility-logs
path: test/s3/versioning/weed-test*.log
@@ -172,7 +172,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-cors-compatibility-logs
path: test/s3/cors/weed-test*.log
@@ -239,7 +239,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-retention-test-logs-${{ matrix.test-type }}
path: test/s3/retention/weed-test*.log
@@ -306,7 +306,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-cors-test-logs-${{ matrix.test-type }}
path: test/s3/cors/weed-test*.log
@@ -355,7 +355,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-retention-worm-logs
path: test/s3/retention/weed-test*.log
@@ -422,7 +422,7 @@ jobs:
- name: Upload stress test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-stress-logs
path: test/s3/versioning/weed-test*.log
@@ -478,7 +478,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-tagging-test-logs
path: test/s3/tagging/weed-test*.log
@@ -531,7 +531,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-remote-cache-test-logs
path: |
+4 -4
View File
@@ -65,7 +65,7 @@ jobs:
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: iam-unit-test-results
path: |
@@ -162,7 +162,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-integration-logs-${{ matrix.test-type }}
path: test/s3/iam/weed-*.log
@@ -222,7 +222,7 @@ jobs:
- name: Upload distributed test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-distributed-logs
path: test/s3/iam/weed-*.log
@@ -274,7 +274,7 @@ jobs:
- name: Upload performance test results
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-performance-results
path: |
+1 -1
View File
@@ -152,7 +152,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-keycloak-test-logs
path: |
+1 -1
View File
@@ -121,7 +121,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: test-logs-python-${{ matrix.python-version }}
path: |
+4 -4
View File
@@ -70,7 +70,7 @@ jobs:
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: policy-unit-test-results
path: |
@@ -178,7 +178,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-policy-variables-test-logs
path: /tmp/weed_policy_test_server.log
@@ -299,7 +299,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
@@ -386,7 +386,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: trusted-proxy-test-logs
path: /tmp/weed_proxy_test.log
+1 -1
View File
@@ -73,7 +73,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-spark-test-logs
path: test/s3/spark/test-output.log
+7 -7
View File
@@ -95,7 +95,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-test-logs-${{ matrix.test-type }}
path: test/s3/sse/weed-test*.log
@@ -143,7 +143,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-compatibility-logs
path: test/s3/sse/weed-test*.log
@@ -192,7 +192,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-metadata-persistence-logs
path: test/s3/sse/weed-test*.log
@@ -241,7 +241,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-copy-operations-logs
path: test/s3/sse/weed-test*.log
@@ -290,7 +290,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-multipart-logs
path: test/s3/sse/weed-test*.log
@@ -340,7 +340,7 @@ jobs:
- name: Upload performance test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-performance-logs
path: test/s3/sse/weed-test*.log
@@ -389,7 +389,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-volume-encryption-logs
path: /tmp/seaweedfs-sse-*.log
+7 -68
View File
@@ -66,7 +66,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-tables-test-logs
path: test/s3tables/table-buckets/test-output.log
@@ -122,7 +122,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: iceberg-catalog-test-logs
path: test/s3tables/catalog/test-output.log
@@ -188,73 +188,12 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: trino-iceberg-catalog-test-logs
path: test/s3tables/catalog_trino/test-output.log
retention-days: 3
polaris-integration-tests:
name: Polaris Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Pre-pull Polaris image
run: docker pull apache/polaris:latest
- name: Run Polaris Integration Tests
timeout-minutes: 25
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Polaris Tests ==="
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
echo "Polaris integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/polaris
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: polaris-test-logs
path: test/s3tables/polaris/test-output.log
retention-days: 3
spark-iceberg-catalog-tests:
name: Spark Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
@@ -315,7 +254,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: spark-iceberg-catalog-test-logs
path: test/s3tables/catalog_spark/test-output.log
@@ -383,7 +322,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: risingwave-catalog-test-logs
path: test/s3tables/catalog_risingwave/test-output.log
@@ -449,7 +388,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: sts-integration-test-logs
path: test/s3tables/sts_integration/test-output.log
@@ -518,7 +457,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: lakekeeper-integration-test-logs
path: test/s3tables/lakekeeper/test-output.log
@@ -125,7 +125,7 @@ jobs:
- name: Upload test results
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: spark-test-results
path: test/java/spark/target/surefire-reports/
@@ -103,7 +103,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: seaweedfs-logs
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
+1 -1
View File
@@ -106,7 +106,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: tus-test-logs
path: |
@@ -90,7 +90,7 @@ jobs:
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: volume-server-integration-test-logs
path: /tmp/volume-server-it-logs/
-1
View File
@@ -7,7 +7,6 @@
[master.maintenance]
# periodically run these scripts are the same as running them from 'weed shell'
# Scripts are skipped while an admin server is connected.
scripts = """
lock
ec.encode -fullPercent=95 -quietFor=1h
+28 -28
View File
@@ -26,7 +26,7 @@ require (
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-redsync/redsync/v4 v4.16.0
github.com/go-redsync/redsync/v4 v4.15.0
github.com/go-sql-driver/mysql v1.9.3
github.com/go-zookeeper/zk v1.0.3 // indirect
github.com/golang/protobuf v1.5.4
@@ -89,20 +89,20 @@ require (
go.etcd.io/etcd/client/v3 v3.6.7
go.mongodb.org/mongo-driver v1.17.6
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.45.0
gocloud.dev/pubsub/natspubsub v0.45.0
gocloud.dev v0.44.0
gocloud.dev/pubsub/natspubsub v0.44.0
gocloud.dev/pubsub/rabbitpubsub v0.44.0
golang.org/x/crypto v0.48.0
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
golang.org/x/image v0.36.0
golang.org/x/net v0.49.0
golang.org/x/oauth2 v0.35.0
golang.org/x/oauth2 v0.34.0
golang.org/x/sys v0.41.0
golang.org/x/text v0.34.0 // indirect
golang.org/x/tools v0.41.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/api v0.258.0
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 // indirect
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
google.golang.org/grpc v1.78.0
google.golang.org/protobuf v1.36.11
gopkg.in/inf.v0 v0.9.1 // indirect
@@ -138,7 +138,8 @@ require (
github.com/hashicorp/raft-boltdb/v2 v2.3.1
github.com/hashicorp/vault/api v1.22.0
github.com/jhump/protoreflect v1.18.0
github.com/linkedin/goavro/v2 v2.15.0
github.com/lib/pq v1.11.1
github.com/linkedin/goavro/v2 v2.14.1
github.com/mattn/go-sqlite3 v1.14.34
github.com/minio/crc64nvme v1.1.1
github.com/orcaman/concurrent-map/v2 v2.0.1
@@ -150,7 +151,7 @@ require (
github.com/redis/go-redis/v9 v9.18.0
github.com/schollz/progressbar/v3 v3.19.0
github.com/seaweedfs/go-fuse/v2 v2.9.1
github.com/shirou/gopsutil/v4 v4.26.2
github.com/shirou/gopsutil/v4 v4.26.1
github.com/tarantool/go-tarantool/v2 v2.4.1
github.com/testcontainers/testcontainers-go v0.39.0
github.com/tikv/client-go/v2 v2.0.7
@@ -171,7 +172,7 @@ require (
atomicgo.dev/keyboard v0.2.9 // indirect
atomicgo.dev/schedule v0.1.0 // indirect
cloud.google.com/go/longrunning v0.7.0 // indirect
cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
dario.cat/mergo v1.0.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
@@ -236,7 +237,6 @@ require (
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.11.1 // indirect
github.com/lithammer/fuzzysearch v1.1.8 // indirect
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
github.com/magiconair/properties v1.8.10 // indirect
@@ -279,10 +279,10 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
go.uber.org/mock v0.5.2 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
@@ -292,12 +292,12 @@ require (
)
require (
cel.dev/expr v0.25.1 // indirect
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go/auth v0.17.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.5.3 // indirect
cloud.google.com/go/monitoring v1.24.3 // indirect
cloud.google.com/go/monitoring v1.24.2 // indirect
filippo.io/edwards25519 v1.1.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
@@ -327,7 +327,7 @@ require (
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -336,11 +336,11 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
github.com/aws/smithy-go v1.24.0
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
@@ -352,7 +352,7 @@ require (
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e // indirect
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
github.com/creasty/defaults v1.8.0 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect
@@ -360,11 +360,11 @@ require (
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
github.com/ebitengine/purego v0.10.0 // indirect
github.com/ebitengine/purego v0.9.1 // indirect
github.com/elastic/gosigar v0.14.3 // indirect
github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -428,8 +428,8 @@ require (
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/nats.go v1.48.0 // indirect
github.com/nats-io/nkeys v0.4.12 // indirect
github.com/nats-io/nats.go v1.43.0 // indirect
github.com/nats-io/nkeys v0.4.11 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/ncw/swift/v2 v2.0.5 // indirect
@@ -493,11 +493,11 @@ require (
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
go.opentelemetry.io/otel v1.40.0 // indirect
go.opentelemetry.io/otel/metric v1.40.0 // indirect
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
go.opentelemetry.io/otel/trace v1.40.0 // indirect
go.opentelemetry.io/otel v1.38.0 // indirect
go.opentelemetry.io/otel/metric v1.38.0 // indirect
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
go.opentelemetry.io/otel/trace v1.38.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect
golang.org/x/term v0.40.0 // indirect
+68 -432
View File
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.15"
appVersion: "4.13"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.15.0
version: 4.0.413
+1 -17
View File
@@ -23,15 +23,8 @@
#
# Adjust storageClass and sizes to match your cluster's available StorageClasses.
# On OpenShift you can discover them with: oc get storageclass
global:
enableReplication: true
# replication type is XYZ:
# X number of replica in other data centers
# Y number of replica in other racks in the same data center
# Z number of replica in other servers in the same rack
replicationPlacement: "000" # no data replica
master:
replicas: 1
data:
type: "persistentVolumeClaim"
size: "10Gi"
@@ -56,7 +49,6 @@ master:
type: RuntimeDefault
volume:
replicas: 1
dataDirs:
- name: data1
type: "persistentVolumeClaim"
@@ -83,7 +75,6 @@ volume:
type: RuntimeDefault
filer:
replicas: 1
data:
type: "persistentVolumeClaim"
size: "25Gi"
@@ -109,19 +100,12 @@ filer:
# S3 gateway (if enabled)
s3:
enabled: true
replicas: 1
port: 8333
enableAuth: true
podSecurityContext:
enabled: true
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
# controller assign them automatically based on the namespace's SCC.
runAsNonRoot: true
logs:
type: "emptyDir"
containerSecurityContext:
enabled: true
allowPrivilegeEscalation: false
@@ -118,10 +118,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.admin "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.admin.extraEnvironmentVars }}
{{- range $key, $value := .Values.admin.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -130,6 +128,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -96,10 +96,8 @@ spec:
- name: WEED_GRPC_CA
value: /usr/local/share/ca-certificates/client/ca.crt
{{- end }}
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.cosi "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.cosi.extraEnvironmentVars }}
{{- range $key, $value := .Values.cosi.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -108,6 +106,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
volumeMounts:
- mountPath: /var/lib/cosi
name: socket
@@ -114,10 +114,8 @@ spec:
optional: true
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.filer "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.filer.extraEnvironmentVars }}
{{- range $key, $value := .Values.filer.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -126,6 +124,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.filer.secretExtraEnvironmentVars }}
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
- name: {{ $key }}
@@ -69,7 +69,9 @@ spec:
priorityClassName: {{ .Values.master.priorityClassName | quote }}
{{- end }}
enableServiceLinks: false
{{- if .Values.global.createClusterRole }}
serviceAccountName: {{ .Values.master.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
{{- end }}
{{- if .Values.master.initContainers }}
initContainers:
{{ tpl .Values.master.initContainers . | nindent 8 | trim }}
@@ -96,10 +98,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.master "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.master.extraEnvironmentVars }}
{{- range $key, $value := .Values.master.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -108,6 +108,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -90,10 +90,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.s3 "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.s3.extraEnvironmentVars }}
{{- range $key, $value := .Values.s3.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -102,6 +100,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -90,10 +90,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.sftp "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.sftp.extraEnvironmentVars }}
{{- range $key, $value := .Values.sftp.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -102,6 +100,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -59,18 +59,6 @@ Inject extra environment vars in the format key:value, if populated
{{- end -}}
{{- end -}}
{{- define "seaweedfs.mergeExtraEnvironmentVars" -}}
{{- $global := ((.global | default dict).extraEnvironmentVars | default dict) -}}
{{- $component := ((.component | default dict).extraEnvironmentVars | default dict) -}}
{{- $target := .target -}}
{{- range $key, $value := $global }}
{{- $_ := set $target $key $value }}
{{- end }}
{{- range $key, $value := $component }}
{{- $_ := set $target $key $value }}
{{- end }}
{{- end -}}
{{/* Return the proper filer image */}}
{{- define "filer.image" -}}
{{- if .Values.filer.imageOverride -}}
@@ -51,7 +51,7 @@ metadata:
annotations:
"helm.sh/hook": post-install,post-upgrade
"helm.sh/hook-weight": "-5"
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
"helm.sh/hook-delete-policy": hook-succeeded
spec:
template:
metadata:
@@ -92,7 +92,6 @@ spec:
- "/bin/sh"
- "-ec"
- |
set -o pipefail
wait_for_service() {
local url=$1
local max_attempts=60 # 5 minutes total (5s * 60)
@@ -118,7 +117,8 @@ spec:
wait_for_service "http://$WEED_CLUSTER_SW_MASTER{{ .Values.master.readinessProbe.httpGet.path }}"
wait_for_service "http://$WEED_CLUSTER_SW_FILER{{ .Values.filer.readinessProbe.httpGet.path }}"
{{- end }}
{{- range $createBuckets }}
set -o pipefail
{{- range $createBuckets }}
{{- $bucketName := .name }}
{{- $bucketLock := or .lock .objectLock .withLock }}
bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; }
@@ -69,7 +69,9 @@ spec:
priorityClassName: {{ $volume.priorityClassName | quote }}
{{- end }}
enableServiceLinks: false
{{- if $.Values.global.createClusterRole }}
serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
{{- end }}
{{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
{{- if $initContainers_exists }}
initContainers:
@@ -116,10 +118,8 @@ spec:
fieldPath: status.hostIP
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" $ }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" $.Values.global "component" $volume "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if $volume.extraEnvironmentVars }}
{{- range $key, $value := $volume.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -128,6 +128,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if $.Values.global.extraEnvironmentVars }}
{{- range $key, $value := $.Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -93,10 +93,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.worker "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.worker.extraEnvironmentVars }}
{{- range $key, $value := .Values.worker.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -105,6 +103,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -100,12 +100,10 @@ message ListEntriesRequest {
string startFromFileName = 3;
bool inclusiveStartFrom = 4;
uint32 limit = 5;
int64 snapshot_ts_ns = 6;
}
message ListEntriesResponse {
Entry entry = 1;
int64 snapshot_ts_ns = 2;
}
message RemoteEntry {
@@ -205,7 +203,6 @@ message CreateEntryRequest {
message CreateEntryResponse {
string error = 1;
SubscribeMetadataResponse metadata_event = 2;
}
message UpdateEntryRequest {
@@ -215,7 +212,6 @@ message UpdateEntryRequest {
repeated int32 signatures = 4;
}
message UpdateEntryResponse {
SubscribeMetadataResponse metadata_event = 1;
}
message AppendToEntryRequest {
@@ -240,7 +236,6 @@ message DeleteEntryRequest {
message DeleteEntryResponse {
string error = 1;
SubscribeMetadataResponse metadata_event = 2;
}
message AtomicRenameEntryRequest {
@@ -474,7 +469,6 @@ message CacheRemoteObjectToLocalClusterRequest {
}
message CacheRemoteObjectToLocalClusterResponse {
Entry entry = 1;
SubscribeMetadataResponse metadata_event = 2;
}
/////////////////////////
@@ -153,38 +153,6 @@ func waitForUrl(t *testing.T, url string, retries int) {
t.Fatalf("Timeout waiting for %s", url)
}
func fetchJSON(url string, out interface{}) error {
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("status %d: %s", resp.StatusCode, string(body))
}
return json.NewDecoder(resp.Body).Decode(out)
}
func mapField(obj map[string]interface{}, key string) (interface{}, bool) {
if obj == nil {
return nil, false
}
if value, ok := obj[key]; ok {
return value, true
}
return nil, false
}
func mapFieldAny(obj map[string]interface{}, keys ...string) (interface{}, bool) {
for _, key := range keys {
if value, ok := mapField(obj, key); ok {
return value, true
}
}
return nil, false
}
func TestEcEndToEnd(t *testing.T) {
defer cleanup()
ensureEnvironment(t)
@@ -194,28 +162,6 @@ func TestEcEndToEnd(t *testing.T) {
// 1. Configure plugin job types for fast EC detection/execution.
t.Log("Configuring plugin job types via API...")
schedulerConfig := map[string]interface{}{
"idle_sleep_seconds": 1,
}
jsonBody, err := json.Marshal(schedulerConfig)
if err != nil {
t.Fatalf("Failed to marshal scheduler config: %v", err)
}
req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/scheduler-config", bytes.NewBuffer(jsonBody))
if err != nil {
t.Fatalf("Failed to create scheduler config request: %v", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Failed to update scheduler config: %v", err)
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("Failed to update scheduler config (status %d): %s", resp.StatusCode, string(body))
}
resp.Body.Close()
// Disable volume balance to reduce interference for this EC-focused test.
balanceConfig := map[string]interface{}{
"job_type": "volume_balance",
@@ -223,16 +169,16 @@ func TestEcEndToEnd(t *testing.T) {
"enabled": false,
},
}
jsonBody, err = json.Marshal(balanceConfig)
jsonBody, err := json.Marshal(balanceConfig)
if err != nil {
t.Fatalf("Failed to marshal volume_balance config: %v", err)
}
req, err = http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody))
req, err := http.NewRequest("PUT", AdminUrl+"/api/plugin/job-types/volume_balance/config", bytes.NewBuffer(jsonBody))
if err != nil {
t.Fatalf("Failed to create volume_balance config request: %v", err)
}
req.Header.Set("Content-Type", "application/json")
resp, err = client.Do(req)
resp, err := client.Do(req)
if err != nil {
t.Fatalf("Failed to update volume_balance config: %v", err)
}
@@ -329,7 +275,6 @@ func TestEcEndToEnd(t *testing.T) {
startTime := time.Now()
ecVerified := false
var lastBody []byte
debugTick := 0
for time.Since(startTime) < 300*time.Second {
// 3.1 Check Master Topology
@@ -355,104 +300,25 @@ func TestEcEndToEnd(t *testing.T) {
}
}
// 3.2 Debug: Check workers, jobs, and scheduler status
debugTick++
var workers []map[string]interface{}
// 3.2 Debug: Check workers and jobs
wResp, wErr := http.Get(AdminUrl + "/api/plugin/workers")
workerCount := 0
ecDetectorCount := 0
ecExecutorCount := 0
if err := fetchJSON(AdminUrl+"/api/plugin/workers", &workers); err == nil {
if wErr == nil {
var workers []interface{}
json.NewDecoder(wResp.Body).Decode(&workers)
wResp.Body.Close()
workerCount = len(workers)
for _, worker := range workers {
capsValue, ok := mapFieldAny(worker, "capabilities", "Capabilities")
if !ok {
continue
}
caps, ok := capsValue.(map[string]interface{})
if !ok {
continue
}
if capValue, ok := caps["erasure_coding"].(map[string]interface{}); ok {
if capValue["can_detect"] == true {
ecDetectorCount++
}
if capValue["can_execute"] == true {
ecExecutorCount++
}
}
}
}
var tasks []map[string]interface{}
tResp, tErr := http.Get(AdminUrl + "/api/plugin/jobs?limit=1000")
taskCount := 0
ecTaskCount := 0
ecTaskStates := map[string]int{}
if err := fetchJSON(AdminUrl+"/api/plugin/jobs?limit=1000", &tasks); err == nil {
if tErr == nil {
var tasks []interface{}
json.NewDecoder(tResp.Body).Decode(&tasks)
tResp.Body.Close()
taskCount = len(tasks)
for _, task := range tasks {
jobType, _ := task["job_type"].(string)
state, _ := task["state"].(string)
if jobType == "erasure_coding" {
ecTaskCount++
ecTaskStates[state]++
}
}
}
t.Logf("Waiting for EC... (Workers: %d det=%d exec=%d, Tasks: %d ec=%d, EC States: %+v)",
workerCount, ecDetectorCount, ecExecutorCount, taskCount, ecTaskCount, ecTaskStates)
if debugTick%3 == 0 {
var pluginStatus map[string]interface{}
if err := fetchJSON(AdminUrl+"/api/plugin/status", &pluginStatus); err == nil {
t.Logf("Plugin status: enabled=%v worker_count=%v worker_grpc_port=%v configured=%v",
pluginStatus["enabled"], pluginStatus["worker_count"], pluginStatus["worker_grpc_port"], pluginStatus["configured"])
}
var schedulerStatus map[string]interface{}
if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-status", &schedulerStatus); err == nil {
if schedValue, ok := schedulerStatus["scheduler"].(map[string]interface{}); ok {
t.Logf("Scheduler status: current_job_type=%v phase=%v last_iteration_had_jobs=%v idle_sleep_seconds=%v last_iteration_done_at=%v next_detection_at=%v",
schedValue["current_job_type"], schedValue["current_phase"],
schedValue["last_iteration_had_jobs"], schedValue["idle_sleep_seconds"], schedValue["last_iteration_done_at"], schedValue["next_detection_at"])
} else {
t.Logf("Scheduler status: %v", schedulerStatus)
}
}
var schedulerStates []map[string]interface{}
if err := fetchJSON(AdminUrl+"/api/plugin/scheduler-states", &schedulerStates); err == nil {
for _, state := range schedulerStates {
if state["job_type"] == "erasure_coding" {
t.Logf("EC scheduler state: enabled=%v detection_in_flight=%v detector_available=%v executor_workers=%v next_detection_at=%v last_run_status=%v last_run_started_at=%v last_run_completed_at=%v",
state["enabled"], state["detection_in_flight"], state["detector_available"],
state["executor_worker_count"], state["next_detection_at"], state["last_run_status"],
state["last_run_started_at"], state["last_run_completed_at"])
break
}
}
}
var jobTypes []map[string]interface{}
if err := fetchJSON(AdminUrl+"/api/plugin/job-types", &jobTypes); err == nil {
var names []string
for _, jobType := range jobTypes {
if name, ok := jobType["job_type"].(string); ok && name != "" {
names = append(names, name)
}
}
t.Logf("Plugin job types: %v", names)
}
var activities []map[string]interface{}
if err := fetchJSON(AdminUrl+"/api/plugin/activities?job_type=erasure_coding&limit=5", &activities); err == nil {
for i := len(activities) - 1; i >= 0; i-- {
act := activities[i]
t.Logf("EC activity: stage=%v message=%v occurred_at=%v", act["stage"], act["message"], act["occurred_at"])
}
}
}
t.Logf("Waiting for EC... (Workers: %d, Active Tasks: %d)", workerCount, taskCount)
time.Sleep(10 * time.Second)
}
+1 -1
View File
@@ -21,7 +21,7 @@
<scala.binary.version>2.12</scala.binary.version>
<junit.version>4.13.2</junit.version>
<seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
<jackson.version>2.18.6</jackson.version> <!-- Upgraded from 2.15.3 -->
<jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
<netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
<parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
<parquet.format.version>2.12.0</parquet.format.version>
@@ -1,573 +0,0 @@
package example
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
v1credentials "github.com/aws/aws-sdk-go/aws/credentials"
v1signer "github.com/aws/aws-sdk-go/aws/signer/v4"
v1s3 "github.com/aws/aws-sdk-go/service/s3"
v2aws "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
v2s3 "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// newS3V2Client creates an AWS SDK v2 S3 client from the test cluster.
func newS3V2Client(cluster *TestCluster) *v2s3.Client {
return v2s3.New(v2s3.Options{
Region: testRegion,
BaseEndpoint: v2aws.String(cluster.s3Endpoint),
Credentials: v2aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, "")),
UsePathStyle: true,
})
}
func TestGetObjectAttributes(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
cluster, err := startMiniCluster(t)
require.NoError(t, err)
defer cluster.Stop()
t.Run("Basic", func(t *testing.T) {
testGetObjectAttributesBasic(t, cluster)
})
t.Run("MultipartObject", func(t *testing.T) {
testGetObjectAttributesMultipart(t, cluster)
})
t.Run("SelectiveAttributes", func(t *testing.T) {
testGetObjectAttributesSelective(t, cluster)
})
t.Run("InvalidAttribute", func(t *testing.T) {
testGetObjectAttributesInvalid(t, cluster)
})
t.Run("NonExistentObject", func(t *testing.T) {
testGetObjectAttributesNotFound(t, cluster)
})
t.Run("VersionedObject", func(t *testing.T) {
testGetObjectAttributesVersioned(t, cluster)
})
t.Run("ConditionalHeaders", func(t *testing.T) {
testGetObjectAttributesConditionalHeaders(t, cluster)
})
t.Run("VersionedConditionalHeaders", func(t *testing.T) {
testGetObjectAttributesVersionedConditionalHeaders(t, cluster)
})
}
func testGetObjectAttributesBasic(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-basic-")
objectKey := "test-object.txt"
objectData := "Hello, GetObjectAttributes!"
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader([]byte(objectData)),
})
require.NoError(t, err)
client := newS3V2Client(cluster)
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String(objectKey),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesEtag,
types.ObjectAttributesStorageClass,
types.ObjectAttributesObjectSize,
types.ObjectAttributesObjectParts,
},
})
require.NoError(t, err)
// ETag should be present and non-empty
require.NotNil(t, resp.ETag)
assert.NotEmpty(t, *resp.ETag)
assert.False(t, strings.Contains(*resp.ETag, `"`), "ETag in XML body should not have quotes")
// ObjectSize should match
require.NotNil(t, resp.ObjectSize)
assert.Equal(t, int64(len(objectData)), *resp.ObjectSize)
// StorageClass should be STANDARD (default)
assert.Equal(t, "STANDARD", string(resp.StorageClass))
// ObjectParts should be nil for non-multipart objects
assert.Nil(t, resp.ObjectParts)
// LastModified header should be present
assert.NotNil(t, resp.LastModified)
t.Logf("Basic GetObjectAttributes passed: ETag=%s, Size=%d, StorageClass=%s",
*resp.ETag, *resp.ObjectSize, resp.StorageClass)
}
func testGetObjectAttributesMultipart(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-mp-")
objectKey := "test-multipart.bin"
// Create a 2-part multipart upload
part1Data := bytes.Repeat([]byte("A"), 5*1024*1024) // 5MB (minimum part size)
part2Data := bytes.Repeat([]byte("B"), 3*1024*1024) // 3MB
initResp, err := cluster.s3Client.CreateMultipartUpload(&v1s3.CreateMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err)
uploadID := initResp.UploadId
part1Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
PartNumber: aws.Int64(1),
UploadId: uploadID,
Body: bytes.NewReader(part1Data),
})
require.NoError(t, err)
part2Resp, err := cluster.s3Client.UploadPart(&v1s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
PartNumber: aws.Int64(2),
UploadId: uploadID,
Body: bytes.NewReader(part2Data),
})
require.NoError(t, err)
_, err = cluster.s3Client.CompleteMultipartUpload(&v1s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: uploadID,
MultipartUpload: &v1s3.CompletedMultipartUpload{
Parts: []*v1s3.CompletedPart{
{ETag: part1Resp.ETag, PartNumber: aws.Int64(1)},
{ETag: part2Resp.ETag, PartNumber: aws.Int64(2)},
},
},
})
require.NoError(t, err)
// Wait briefly for metadata to settle
time.Sleep(200 * time.Millisecond)
client := newS3V2Client(cluster)
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String(objectKey),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesObjectParts,
types.ObjectAttributesObjectSize,
},
})
require.NoError(t, err)
require.NotNil(t, resp.ObjectSize)
assert.Equal(t, int64(len(part1Data)+len(part2Data)), *resp.ObjectSize)
require.NotNil(t, resp.ObjectParts, "ObjectParts should be present for multipart objects")
assert.Equal(t, int32(2), *resp.ObjectParts.TotalPartsCount)
require.Len(t, resp.ObjectParts.Parts, 2)
assert.Equal(t, int32(1), *resp.ObjectParts.Parts[0].PartNumber)
assert.Equal(t, int64(len(part1Data)), *resp.ObjectParts.Parts[0].Size)
assert.Equal(t, int32(2), *resp.ObjectParts.Parts[1].PartNumber)
assert.Equal(t, int64(len(part2Data)), *resp.ObjectParts.Parts[1].Size)
// Test pagination: MaxParts=1
resp2, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String(objectKey),
MaxParts: v2aws.Int32(1),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesObjectParts,
},
})
require.NoError(t, err)
require.NotNil(t, resp2.ObjectParts)
assert.Len(t, resp2.ObjectParts.Parts, 1)
assert.True(t, *resp2.ObjectParts.IsTruncated)
assert.Equal(t, int32(2), *resp2.ObjectParts.TotalPartsCount)
t.Logf("Multipart GetObjectAttributes passed: %d parts, total size %d",
*resp.ObjectParts.TotalPartsCount, *resp.ObjectSize)
}
func testGetObjectAttributesSelective(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-sel-")
objectKey := "test-selective.txt"
objectData := "Selective attributes test"
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader([]byte(objectData)),
})
require.NoError(t, err)
client := newS3V2Client(cluster)
// Request only ETag
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String(objectKey),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesEtag,
},
})
require.NoError(t, err)
require.NotNil(t, resp.ETag)
assert.NotEmpty(t, *resp.ETag)
assert.Nil(t, resp.ObjectSize, "ObjectSize should not be present when not requested")
assert.Empty(t, string(resp.StorageClass), "StorageClass should not be present when not requested")
assert.Nil(t, resp.ObjectParts, "ObjectParts should not be present when not requested")
t.Logf("Selective GetObjectAttributes passed: ETag=%s", *resp.ETag)
}
func testGetObjectAttributesInvalid(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-inv-")
objectKey := "test-object.txt"
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader([]byte("test")),
})
require.NoError(t, err)
// Use raw HTTP to send an invalid attribute name since the SDK validates
reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
req, err := http.NewRequest("GET", reqURL, nil)
require.NoError(t, err)
req.Header.Set("X-Amz-Object-Attributes", "InvalidAttr")
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
require.NoError(t, err)
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 400, resp.StatusCode)
t.Logf("Invalid attribute test passed: got %d", resp.StatusCode)
}
func testGetObjectAttributesNotFound(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-nf-")
client := newS3V2Client(cluster)
_, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("nonexistent-key"),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesEtag,
},
})
require.Error(t, err)
assert.Contains(t, err.Error(), "NoSuchKey")
t.Logf("NotFound GetObjectAttributes passed")
}
func testGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster) {
client := newS3V2Client(cluster)
bucketName := createTestBucket(t, cluster, "test-goa-ver-")
// Enable versioning
_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
Bucket: v2aws.String(bucketName),
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
require.NoError(t, err)
time.Sleep(200 * time.Millisecond)
// Put two versions of the same object
v1Data := "version 1 content"
putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("versioned-key"),
Body: strings.NewReader(v1Data),
})
require.NoError(t, err)
require.NotNil(t, putResp1.VersionId)
versionId1 := *putResp1.VersionId
v2Data := "version 2 content - longer"
putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("versioned-key"),
Body: strings.NewReader(v2Data),
})
require.NoError(t, err)
require.NotNil(t, putResp2.VersionId)
versionId2 := *putResp2.VersionId
assert.NotEqual(t, versionId1, versionId2, "versions should differ")
// GetObjectAttributes for latest version (v2)
resp, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("versioned-key"),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesObjectSize,
types.ObjectAttributesEtag,
},
})
require.NoError(t, err)
require.NotNil(t, resp.ObjectSize)
assert.Equal(t, int64(len(v2Data)), *resp.ObjectSize)
require.NotNil(t, resp.VersionId)
assert.Equal(t, versionId2, *resp.VersionId)
// GetObjectAttributes for specific older version (v1)
resp1, err := client.GetObjectAttributes(context.Background(), &v2s3.GetObjectAttributesInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("versioned-key"),
VersionId: v2aws.String(versionId1),
ObjectAttributes: []types.ObjectAttributes{
types.ObjectAttributesObjectSize,
types.ObjectAttributesEtag,
},
})
require.NoError(t, err)
require.NotNil(t, resp1.ObjectSize)
assert.Equal(t, int64(len(v1Data)), *resp1.ObjectSize)
require.NotNil(t, resp1.VersionId)
assert.Equal(t, versionId1, *resp1.VersionId)
t.Logf("Versioned GetObjectAttributes passed: v1 size=%d (id=%s), v2 size=%d (id=%s)",
*resp1.ObjectSize, versionId1, *resp.ObjectSize, versionId2)
}
// signedGetObjectAttributes creates a signed GET request for ?attributes with custom headers.
func signedGetObjectAttributes(t *testing.T, cluster *TestCluster, bucketName, objectKey string, extraHeaders map[string]string) *http.Response {
reqURL := fmt.Sprintf("%s/%s/%s?attributes", cluster.s3Endpoint, bucketName, objectKey)
req, err := http.NewRequest("GET", reqURL, nil)
require.NoError(t, err)
req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
for k, v := range extraHeaders {
req.Header.Set(k, v)
}
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
require.NoError(t, err)
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
require.NoError(t, err)
return resp
}
func testGetObjectAttributesConditionalHeaders(t *testing.T, cluster *TestCluster) {
bucketName := createTestBucket(t, cluster, "test-goa-cond-")
objectKey := "cond-test.txt"
_, err := cluster.s3Client.PutObject(&v1s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader([]byte("conditional headers test")),
})
require.NoError(t, err)
// Get the ETag and Last-Modified for the object
headResp, err := cluster.s3Client.HeadObject(&v1s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
require.NoError(t, err)
etag := aws.StringValue(headResp.ETag)
lastModified := headResp.LastModified
require.NotNil(t, lastModified)
pastDate := lastModified.Add(-1 * time.Hour).UTC().Format(http.TimeFormat)
futureDate := lastModified.Add(1 * time.Hour).UTC().Format(http.TimeFormat)
// RFC 7232: If-Match true + If-Unmodified-Since false => 200 OK
// If-Unmodified-Since is ignored when If-Match is present
t.Run("IfMatch_true_IfUnmodifiedSince_false", func(t *testing.T) {
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
"If-Match": etag,
"If-Unmodified-Since": pastDate, // object was modified after this => false
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 200, resp.StatusCode,
"If-Match=true should return 200 even when If-Unmodified-Since=false (RFC 7232 Section 3.4)")
})
// RFC 7232: If-None-Match false + If-Modified-Since true => 304 Not Modified
// If-Modified-Since is ignored when If-None-Match is present
t.Run("IfNoneMatch_false_IfModifiedSince_true", func(t *testing.T) {
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
"If-None-Match": etag,
"If-Modified-Since": pastDate, // object was modified after this => true
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 304, resp.StatusCode,
"If-None-Match=false (ETag match) should return 304 even when If-Modified-Since=true (RFC 7232 Section 3.3)")
})
// If-Match succeeds, If-Unmodified-Since also succeeds => 200
t.Run("IfMatch_true_IfUnmodifiedSince_true", func(t *testing.T) {
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
"If-Match": etag,
"If-Unmodified-Since": futureDate,
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 200, resp.StatusCode)
})
// If-None-Match passes (ETag differs), If-Modified-Since ignored => 200
// Per RFC 7232, If-Modified-Since is ignored when If-None-Match is present
t.Run("IfNoneMatch_true_IfModifiedSince_ignored", func(t *testing.T) {
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
"If-None-Match": `"nonexistent-etag"`,
"If-Modified-Since": futureDate, // would fail alone, but is ignored
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 200, resp.StatusCode,
"If-None-Match=true means If-Modified-Since is ignored, should return 200 (RFC 7232 Section 3.3)")
})
// If-Match fails => 412 regardless of If-Unmodified-Since
t.Run("IfMatch_false", func(t *testing.T) {
resp := signedGetObjectAttributes(t, cluster, bucketName, objectKey, map[string]string{
"If-Match": `"wrong-etag"`,
"If-Unmodified-Since": futureDate,
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 412, resp.StatusCode)
})
t.Logf("Conditional headers tests passed")
}
// signedGetObjectAttributesVersioned creates a signed GET request for ?attributes&versionId=... with custom headers.
func signedGetObjectAttributesVersioned(t *testing.T, cluster *TestCluster, bucketName, objectKey, versionId string, extraHeaders map[string]string) *http.Response {
reqURL := fmt.Sprintf("%s/%s/%s?attributes&versionId=%s", cluster.s3Endpoint, bucketName, objectKey, versionId)
req, err := http.NewRequest("GET", reqURL, nil)
require.NoError(t, err)
req.Header.Set("X-Amz-Object-Attributes", "ETag,ObjectSize")
for k, v := range extraHeaders {
req.Header.Set(k, v)
}
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
_, err = signer.Sign(req, nil, "s3", testRegion, time.Now())
require.NoError(t, err)
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
require.NoError(t, err)
return resp
}
func testGetObjectAttributesVersionedConditionalHeaders(t *testing.T, cluster *TestCluster) {
client := newS3V2Client(cluster)
bucketName := createTestBucket(t, cluster, "test-goa-vcond-")
// Enable versioning
_, err := client.PutBucketVersioning(context.Background(), &v2s3.PutBucketVersioningInput{
Bucket: v2aws.String(bucketName),
VersioningConfiguration: &types.VersioningConfiguration{
Status: types.BucketVersioningStatusEnabled,
},
})
require.NoError(t, err)
time.Sleep(200 * time.Millisecond)
// Put two versions with different content (different ETags)
v1Data := "version 1 - original"
putResp1, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("vcond-key"),
Body: strings.NewReader(v1Data),
})
require.NoError(t, err)
require.NotNil(t, putResp1.VersionId)
vid1 := *putResp1.VersionId
v2Data := "version 2 - updated content"
putResp2, err := client.PutObject(context.Background(), &v2s3.PutObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("vcond-key"),
Body: strings.NewReader(v2Data),
})
require.NoError(t, err)
require.NotNil(t, putResp2.VersionId)
vid2 := *putResp2.VersionId
// Get ETags for each version
headV1, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("vcond-key"),
VersionId: v2aws.String(vid1),
})
require.NoError(t, err)
etagV1 := *headV1.ETag
headV2, err := client.HeadObject(context.Background(), &v2s3.HeadObjectInput{
Bucket: v2aws.String(bucketName),
Key: v2aws.String("vcond-key"),
VersionId: v2aws.String(vid2),
})
require.NoError(t, err)
etagV2 := *headV2.ETag
require.NotEqual(t, etagV1, etagV2, "versions should have different ETags")
// If-Match with v1's ETag + versionId=v1 => 200
// Before the fix, this would fail with 412 because conditional headers
// were evaluated against the latest version (v2) whose ETag differs
t.Run("IfMatch_v1_etag_versionId_v1", func(t *testing.T) {
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
"If-Match": etagV1,
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 200, resp.StatusCode,
"If-Match with v1 ETag targeting versionId=v1 should return 200")
})
// If-Match with v2's ETag + versionId=v1 => 412
// The ETag doesn't match v1, so this should fail
t.Run("IfMatch_v2_etag_versionId_v1", func(t *testing.T) {
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
"If-Match": etagV2,
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 412, resp.StatusCode,
"If-Match with v2 ETag targeting versionId=v1 should return 412")
})
// If-None-Match with v1's ETag + versionId=v1 => 304
t.Run("IfNoneMatch_v1_etag_versionId_v1", func(t *testing.T) {
resp := signedGetObjectAttributesVersioned(t, cluster, bucketName, "vcond-key", vid1, map[string]string{
"If-None-Match": etagV1,
})
defer resp.Body.Close()
io.Copy(io.Discard, resp.Body)
assert.Equal(t, 304, resp.StatusCode,
"If-None-Match with v1 ETag targeting versionId=v1 should return 304")
})
t.Logf("Versioned conditional headers tests passed: vid1=%s, vid2=%s", vid1, vid2)
}
-212
View File
@@ -422,218 +422,6 @@ func TestS3MultipartOperationsInheritPutObjectPermissions(t *testing.T) {
require.Equal(t, 0, len(listUploadsOut.Uploads))
}
// TestS3IAMManagedPolicyLifecycle is an end-to-end integration test covering the
// user-reported use case in https://github.com/seaweedfs/seaweedfs/issues/8506
// where managed policy operations (GetPolicy, ListPolicies, DeletePolicy,
// AttachUserPolicy, DetachUserPolicy) returned 500 errors.
func TestS3IAMManagedPolicyLifecycle(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
cluster, err := startMiniCluster(t)
require.NoError(t, err)
defer cluster.Stop()
iamClient := newIAMClient(t, cluster.s3Endpoint)
// Step 1: Create a user (this already worked per the issue)
userName := uniqueName("lifecycle-user")
_, err = iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
require.NoError(t, err, "CreateUser should succeed")
// Step 2: Create a managed policy via IAM API
policyName := uniqueName("lifecycle-policy")
policyArn := fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)
policyDoc := `{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject"],
"Resource": "arn:aws:s3:::*"
}]
}`
createOut, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
PolicyName: aws.String(policyName),
PolicyDocument: aws.String(policyDoc),
})
require.NoError(t, err, "CreatePolicy should succeed")
require.NotNil(t, createOut.Policy)
require.Equal(t, policyName, *createOut.Policy.PolicyName)
// Step 3: ListPolicies — should include the created policy (was returning 500)
listOut, err := iamClient.ListPolicies(&iam.ListPoliciesInput{})
require.NoError(t, err, "ListPolicies should succeed (was returning 500)")
require.True(t, managedPolicyContains(listOut.Policies, policyName),
"ListPolicies should contain the newly created policy")
// Step 4: GetPolicy by ARN — should return the policy (was returning 500)
getOut, err := iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
require.NoError(t, err, "GetPolicy should succeed (was returning 500)")
require.NotNil(t, getOut.Policy)
require.Equal(t, policyName, *getOut.Policy.PolicyName)
require.Equal(t, policyArn, *getOut.Policy.Arn)
// Step 5: AttachUserPolicy — should succeed (was returning 500)
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String(policyArn),
})
require.NoError(t, err, "AttachUserPolicy should succeed (was returning 500)")
// Step 6: ListAttachedUserPolicies — verify the policy is attached
attachedOut, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
UserName: aws.String(userName),
})
require.NoError(t, err, "ListAttachedUserPolicies should succeed")
require.True(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
"Policy should appear in user's attached policies")
// Step 7: Idempotent re-attach should not fail
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String(policyArn),
})
require.NoError(t, err, "Re-attaching same policy should be idempotent")
// Step 8: DeletePolicy while attached — should fail with DeleteConflict (AWS behavior)
_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
require.Error(t, err, "DeletePolicy should fail while policy is attached")
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeDeleteConflictException, awsErr.Code(),
"Should return DeleteConflict when deleting attached policy")
// Step 9: DetachUserPolicy
_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String(policyArn),
})
require.NoError(t, err, "DetachUserPolicy should succeed")
// Verify detached
attachedOut, err = iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
UserName: aws.String(userName),
})
require.NoError(t, err)
require.False(t, attachedPolicyContains(attachedOut.AttachedPolicies, policyName),
"Policy should no longer appear in user's attached policies after detach")
// Step 10: DeletePolicy — should now succeed (was returning XML parsing error)
_, err = iamClient.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: aws.String(policyArn)})
require.NoError(t, err, "DeletePolicy should succeed after detach (was returning XML parsing error)")
// Step 11: Verify the policy is gone
listOut, err = iamClient.ListPolicies(&iam.ListPoliciesInput{})
require.NoError(t, err)
require.False(t, managedPolicyContains(listOut.Policies, policyName),
"Deleted policy should not appear in ListPolicies")
_, err = iamClient.GetPolicy(&iam.GetPolicyInput{PolicyArn: aws.String(policyArn)})
require.Error(t, err, "GetPolicy should fail for deleted policy")
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
}
// TestS3IAMManagedPolicyErrorCases covers error cases from the user-reported issue:
// invalid ARNs, missing policies, and missing users.
func TestS3IAMManagedPolicyErrorCases(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
cluster, err := startMiniCluster(t)
require.NoError(t, err)
defer cluster.Stop()
iamClient := newIAMClient(t, cluster.s3Endpoint)
t.Run("GetPolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
_, err := iamClient.GetPolicy(&iam.GetPolicyInput{
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
t.Run("DeletePolicy with nonexistent ARN returns NoSuchEntity", func(t *testing.T) {
_, err := iamClient.DeletePolicy(&iam.DeletePolicyInput{
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
t.Run("AttachUserPolicy with nonexistent policy returns NoSuchEntity", func(t *testing.T) {
userName := uniqueName("err-user")
_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
require.NoError(t, err)
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String("arn:aws:iam:::policy/does-not-exist"),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
t.Run("AttachUserPolicy with nonexistent user returns NoSuchEntity", func(t *testing.T) {
policyName := uniqueName("err-policy")
_, err := iamClient.CreatePolicy(&iam.CreatePolicyInput{
PolicyName: aws.String(policyName),
PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
})
require.NoError(t, err)
_, err = iamClient.AttachUserPolicy(&iam.AttachUserPolicyInput{
UserName: aws.String("nonexistent-user"),
PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
t.Run("DetachUserPolicy that is not attached returns NoSuchEntity", func(t *testing.T) {
userName := uniqueName("detach-user")
_, err := iamClient.CreateUser(&iam.CreateUserInput{UserName: aws.String(userName)})
require.NoError(t, err)
policyName := uniqueName("detach-policy")
_, err = iamClient.CreatePolicy(&iam.CreatePolicyInput{
PolicyName: aws.String(policyName),
PolicyDocument: aws.String(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"*"}]}`),
})
require.NoError(t, err)
_, err = iamClient.DetachUserPolicy(&iam.DetachUserPolicyInput{
UserName: aws.String(userName),
PolicyArn: aws.String(fmt.Sprintf("arn:aws:iam:::policy/%s", policyName)),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
t.Run("ListAttachedUserPolicies for nonexistent user returns NoSuchEntity", func(t *testing.T) {
_, err := iamClient.ListAttachedUserPolicies(&iam.ListAttachedUserPoliciesInput{
UserName: aws.String("nonexistent-user"),
})
require.Error(t, err)
var awsErr awserr.Error
require.True(t, errors.As(err, &awsErr))
require.Equal(t, iam.ErrCodeNoSuchEntityException, awsErr.Code())
})
}
func execShell(t *testing.T, weedCmd, master, filer, shellCmd string) string {
// weed shell -master=... -filer=...
args := []string{"shell", "-master=" + master, "-filer=" + filer}
+118 -466
View File
@@ -1,32 +1,15 @@
package lakekeeper
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
type TestEnvironment struct {
@@ -48,19 +31,15 @@ type TestEnvironment struct {
secretKey string
}
const (
lakekeeperRegion = "us-east-1"
lakekeeperRoleArn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
lakekeeperSessionName = "lakekeeper-session"
)
type lakekeeperSession struct {
endpoint string
region string
creds aws.Credentials
}
func TestLakekeeperIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !testutil.HasDocker() {
t.Skip("Docker not available, skipping Lakekeeper integration test")
}
env := NewTestEnvironment(t)
defer env.Cleanup(t)
@@ -72,18 +51,6 @@ func TestLakekeeperIntegration(t *testing.T) {
runLakekeeperRepro(t, env)
}
func TestLakekeeperTableBucketIntegration(t *testing.T) {
env := NewTestEnvironment(t)
defer env.Cleanup(t)
fmt.Printf(">>> Starting SeaweedFS with Lakekeeper configuration...\n")
env.StartSeaweedFS(t)
fmt.Printf(">>> SeaweedFS started.\n")
// Run python script in docker to test STS and S3 Tables operations
runLakekeeperTableBucketRepro(t, env)
}
func NewTestEnvironment(t *testing.T) *TestEnvironment {
t.Helper()
@@ -251,439 +218,124 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
func runLakekeeperRepro(t *testing.T, env *TestEnvironment) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
scriptContent := fmt.Sprintf(`
import boto3
import botocore.config
import botocore
from botocore.exceptions import ClientError
import os
import sys
import time
import logging
session, err := newLakekeeperSession(ctx, env)
if err != nil {
t.Fatalf("AssumeRole failed: %v", err)
# Enable botocore debug logging to see signature calculation
logging.basicConfig(level=logging.DEBUG)
botocore.session.get_session().set_debug_logger()
print("Starting Lakekeeper repro test...")
endpoint_url = "http://host.docker.internal:%d"
access_key = "%s"
secret_key = "%s"
region = "us-east-1"
print(f"Connecting to {endpoint_url}")
try:
config = botocore.config.Config(
retries={'max_attempts': 3}
)
sts = boto3.client(
'sts',
endpoint_url=endpoint_url,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region,
config=config
)
role_arn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
session_name = "lakekeeper-session"
print(f"Calling AssumeRole on {role_arn} with POST body...")
# Standard boto3 call sends parameters in POST body
response = sts.assume_role(
RoleArn=role_arn,
RoleSessionName=session_name
)
creds = response['Credentials']
access_key_id = creds['AccessKeyId']
secret_access_key = creds['SecretAccessKey']
session_token = creds['SessionToken']
print(f"Success! Got credentials with prefix: {access_key_id[:4]}")
if not access_key_id.startswith("ASIA"):
print(f"FAILED: Expected ASIA prefix, got {access_key_id}")
sys.exit(1)
print("Verifying S3 operations with vended credentials...")
s3 = boto3.client(
's3',
endpoint_url=endpoint_url,
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
aws_session_token=session_token,
region_name=region,
config=config
)
bucket = "lakekeeper-vended-bucket"
print(f"Creating bucket {bucket}...")
s3.create_bucket(Bucket=bucket)
print("Listing buckets...")
response = s3.list_buckets()
buckets = [b['Name'] for b in response['Buckets']]
print(f"Found buckets: {buckets}")
if bucket not in buckets:
print(f"FAILED: Bucket {bucket} not found in list")
sys.exit(1)
print("SUCCESS: Lakekeeper flow verified!")
sys.exit(0)
except Exception as e:
print(f"FAILED: {e}")
# Print more details if it is a ClientError
if hasattr(e, 'response'):
print(f"Response: {e.response}")
sys.exit(1)
`, env.s3Port, env.accessKey, env.secretKey)
scriptPath := filepath.Join(env.dataDir, "lakekeeper_repro.py")
if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
t.Fatalf("Failed to write python script: %v", err)
}
s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
if err != nil {
t.Fatalf("Create S3 client failed: %v", err)
}
containerName := "seaweed-lakekeeper-client-" + fmt.Sprintf("%d", time.Now().UnixNano())
bucketName := fmt.Sprintf("lakekeeper-vended-bucket-%d", time.Now().UnixNano())
if _, err := s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
}); err != nil {
t.Fatalf("CreateBucket failed: %v", err)
}
// Create a context with timeout for the docker run command
dockerCtx, dockerCancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer dockerCancel()
bucketCreated := true
defer func() {
if !bucketCreated {
return
}
_, _ = s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
Bucket: aws.String(bucketName),
})
}()
listResp, err := s3Client.ListBuckets(ctx, &s3.ListBucketsInput{})
if err != nil {
t.Fatalf("ListBuckets failed: %v", err)
}
found := false
for _, bucket := range listResp.Buckets {
if aws.ToString(bucket.Name) == bucketName {
found = true
break
}
}
if !found {
t.Fatalf("Bucket %s not found in list", bucketName)
}
if _, err := s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
Bucket: aws.String(bucketName),
}); err != nil {
t.Fatalf("DeleteBucket failed: %v", err)
}
bucketCreated = false
}
func runLakekeeperTableBucketRepro(t *testing.T, env *TestEnvironment) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
session, err := newLakekeeperSession(ctx, env)
if err != nil {
t.Fatalf("AssumeRole failed: %v", err)
}
client := newS3TablesClient(session.endpoint, session.region, session.creds)
bucketName := fmt.Sprintf("lakekeeper-table-bucket-%d", time.Now().UnixNano())
bucketARN, err := client.CreateTableBucket(ctx, bucketName)
if err != nil {
t.Fatalf("CreateTableBucket failed: %v", err)
}
bucketDeleted := false
namespaceCreated := false
tableCreated := false
namespaceName := fmt.Sprintf("lakekeeper_ns_%d", time.Now().UnixNano())
tableName := fmt.Sprintf("lakekeeper_table_%d", time.Now().UnixNano())
defer func() {
if tableCreated {
if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
t.Logf("Failed to delete table: %v", err)
}
}
if namespaceCreated {
if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
t.Logf("Failed to delete namespace: %v", err)
}
}
if bucketDeleted {
return
}
if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
t.Logf("Failed to delete table bucket: %v", err)
}
}()
buckets, err := client.ListTableBuckets(ctx)
if err != nil {
t.Fatalf("ListTableBuckets failed: %v", err)
}
found := false
for _, b := range buckets {
if b.Name == bucketName {
found = true
break
}
}
if !found {
t.Fatalf("Created table bucket %s not found in list", bucketName)
}
if _, err := client.GetTableBucket(ctx, bucketARN); err != nil {
t.Fatalf("GetTableBucket failed: %v", err)
}
if err := client.CreateNamespace(ctx, bucketARN, namespaceName); err != nil {
t.Fatalf("CreateNamespace failed: %v", err)
}
namespaceCreated = true
if err := client.CreateTable(ctx, bucketARN, namespaceName, tableName); err != nil {
t.Fatalf("CreateTable failed: %v", err)
}
tableCreated = true
s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
if err != nil {
t.Fatalf("Create S3 client failed: %v", err)
}
objectKey := fmt.Sprintf("%s/%s/data/part-%d.parquet", namespaceName, tableName, time.Now().UnixNano())
createResp, err := s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("CreateMultipartUpload failed: %v", err)
}
uploadID := aws.ToString(createResp.UploadId)
multipartCompleted := false
defer func() {
if uploadID == "" || multipartCompleted {
return
}
_, _ = s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
})
}()
partSize := 5 * 1024 * 1024
part1 := bytes.Repeat([]byte("a"), partSize)
part2 := bytes.Repeat([]byte("b"), 1024*1024)
part1Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(1),
Body: bytes.NewReader(part1),
})
if err != nil {
t.Fatalf("UploadPart 1 failed: %v", err)
}
part2Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(2),
Body: bytes.NewReader(part2),
})
if err != nil {
t.Fatalf("UploadPart 2 failed: %v", err)
}
_, err = s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
MultipartUpload: &s3types.CompletedMultipartUpload{
Parts: []s3types.CompletedPart{
{
ETag: part1Resp.ETag,
PartNumber: aws.Int32(1),
},
{
ETag: part2Resp.ETag,
PartNumber: aws.Int32(2),
},
},
},
})
if err != nil {
t.Fatalf("CompleteMultipartUpload failed: %v", err)
}
multipartCompleted = true
headResp, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("HeadObject after multipart upload failed: %v", err)
}
expectedSize := int64(len(part1) + len(part2))
if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
}
if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
t.Fatalf("DeleteTable failed: %v", err)
}
tableCreated = false
if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
t.Fatalf("DeleteNamespace failed: %v", err)
}
namespaceCreated = false
if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
t.Fatalf("DeleteTableBucket failed: %v", err)
}
bucketDeleted = true
if _, err := client.GetTableBucket(ctx, bucketARN); err == nil {
t.Fatalf("expected GetTableBucket to fail after deletion")
}
}
type s3TablesClient struct {
endpoint string
region string
creds aws.Credentials
httpClient *http.Client
}
func newS3TablesClient(endpoint, region string, creds aws.Credentials) *s3TablesClient {
return &s3TablesClient{
endpoint: endpoint,
region: region,
creds: creds,
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
func (c *s3TablesClient) CreateTableBucket(ctx context.Context, name string) (string, error) {
req := &s3tables.CreateTableBucketRequest{Name: name}
var resp s3tables.CreateTableBucketResponse
if err := c.doRequest(ctx, "CreateTableBucket", http.MethodPut, "/buckets", req, &resp); err != nil {
return "", err
}
return resp.ARN, nil
}
func (c *s3TablesClient) GetTableBucket(ctx context.Context, arn string) (*s3tables.GetTableBucketResponse, error) {
path := "/buckets/" + url.PathEscape(arn)
var resp s3tables.GetTableBucketResponse
if err := c.doRequest(ctx, "GetTableBucket", http.MethodGet, path, nil, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func (c *s3TablesClient) ListTableBuckets(ctx context.Context) ([]s3tables.TableBucketSummary, error) {
var resp s3tables.ListTableBucketsResponse
if err := c.doRequest(ctx, "ListTableBuckets", http.MethodGet, "/buckets", nil, &resp); err != nil {
return nil, err
}
return resp.TableBuckets, nil
}
func (c *s3TablesClient) DeleteTableBucket(ctx context.Context, arn string) error {
path := "/buckets/" + url.PathEscape(arn)
return c.doRequest(ctx, "DeleteTableBucket", http.MethodDelete, path, nil, nil)
}
func (c *s3TablesClient) CreateNamespace(ctx context.Context, bucketARN, namespace string) error {
req := &s3tables.CreateNamespaceRequest{
Namespace: []string{namespace},
}
path := "/namespaces/" + url.PathEscape(bucketARN)
return c.doRequest(ctx, "CreateNamespace", http.MethodPut, path, req, nil)
}
func (c *s3TablesClient) DeleteNamespace(ctx context.Context, bucketARN, namespace string) error {
path := "/namespaces/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
return c.doRequest(ctx, "DeleteNamespace", http.MethodDelete, path, nil, nil)
}
func (c *s3TablesClient) CreateTable(ctx context.Context, bucketARN, namespace, name string) error {
req := &s3tables.CreateTableRequest{
Name: name,
Format: "ICEBERG",
}
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
return c.doRequest(ctx, "CreateTable", http.MethodPut, path, req, nil)
}
func (c *s3TablesClient) DeleteTable(ctx context.Context, bucketARN, namespace, name string) error {
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace) + "/" + url.PathEscape(name)
return c.doRequest(ctx, "DeleteTable", http.MethodDelete, path, nil, nil)
}
func (c *s3TablesClient) doRequest(ctx context.Context, operation, method, path string, body interface{}, out interface{}) error {
var bodyBytes []byte
if body != nil {
encoded, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("%s: marshal request: %w", operation, err)
}
bodyBytes = encoded
}
req, err := http.NewRequestWithContext(ctx, method, c.endpoint+path, bytes.NewReader(bodyBytes))
if err != nil {
return fmt.Errorf("%s: create request: %w", operation, err)
}
if body != nil {
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
}
req.Host = req.URL.Host
req.Header.Set("Host", req.URL.Host)
payloadHash := sha256.Sum256(bodyBytes)
if err := v4.NewSigner().SignHTTP(ctx, c.creds, req, hex.EncodeToString(payloadHash[:]), "s3tables", c.region, time.Now()); err != nil {
return fmt.Errorf("%s: sign request: %w", operation, err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("%s: request failed: %w", operation, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
bodyBytes, readErr := io.ReadAll(resp.Body)
if readErr != nil {
return fmt.Errorf("%s failed with status %d and could not read response body: %v", operation, resp.StatusCode, readErr)
}
var errResp s3tables.S3TablesError
if jsonErr := json.Unmarshal(bodyBytes, &errResp); jsonErr == nil && (errResp.Type != "" || errResp.Message != "") {
return fmt.Errorf("%s failed: %s - %s", operation, errResp.Type, errResp.Message)
}
return fmt.Errorf("%s failed with status %d: %s", operation, resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
}
if out == nil {
return nil
}
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
return fmt.Errorf("%s: decode response: %w", operation, err)
}
return nil
}
func assumeRole(ctx context.Context, endpoint, region, accessKey, secretKey, roleArn, sessionName string) (aws.Credentials, error) {
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == sts.ServiceID {
return aws.Endpoint{
URL: endpoint,
SigningRegion: region,
HostnameImmutable: true,
}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
config.WithEndpointResolverWithOptions(resolver),
cmd := exec.CommandContext(dockerCtx, "docker", "run", "--rm",
"--name", containerName,
"--add-host", "host.docker.internal:host-gateway",
"-v", fmt.Sprintf("%s:/work", env.dataDir),
"python:3",
"/bin/bash", "-c", "pip install boto3 && python /work/lakekeeper_repro.py",
)
if err != nil {
return aws.Credentials{}, err
}
client := sts.NewFromConfig(cfg)
resp, err := client.AssumeRole(ctx, &sts.AssumeRoleInput{
RoleArn: aws.String(roleArn),
RoleSessionName: aws.String(sessionName),
})
output, err := cmd.CombinedOutput()
if err != nil {
return aws.Credentials{}, err
}
if resp.Credentials == nil {
return aws.Credentials{}, fmt.Errorf("missing credentials in AssumeRole response")
}
return aws.Credentials{
AccessKeyID: aws.ToString(resp.Credentials.AccessKeyId),
SecretAccessKey: aws.ToString(resp.Credentials.SecretAccessKey),
SessionToken: aws.ToString(resp.Credentials.SessionToken),
Source: "lakekeeper-sts",
}, nil
}
func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials) (*s3.Client, error) {
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID {
return aws.Endpoint{
URL: endpoint,
SigningRegion: region,
HostnameImmutable: true,
}, nil
if dockerCtx.Err() == context.DeadlineExceeded {
t.Fatalf("Lakekeeper repro client timed out after 5 minutes\nOutput:\n%s", string(output))
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
config.WithEndpointResolverWithOptions(resolver),
)
if err != nil {
return nil, err
t.Fatalf("Lakekeeper repro client failed: %v\nOutput:\n%s", err, string(output))
}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = true
}), nil
}
func newLakekeeperSession(ctx context.Context, env *TestEnvironment) (lakekeeperSession, error) {
endpoint := fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
creds, err := assumeRole(ctx, endpoint, lakekeeperRegion, env.accessKey, env.secretKey, lakekeeperRoleArn, lakekeeperSessionName)
if err != nil {
return lakekeeperSession{}, err
}
return lakekeeperSession{
endpoint: endpoint,
region: lakekeeperRegion,
creds: creds,
}, nil
t.Logf("Lakekeeper repro client output:\n%s", string(output))
}
-377
View File
@@ -1,377 +0,0 @@
package polaris
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
)
const (
polarisImage = "apache/polaris:latest"
polarisRealm = "POLARIS"
polarisRootClientID = "root"
polarisRootClientSecret = "s3cr3t"
polarisRegion = "us-east-1"
polarisRoleArn = "arn:aws:iam::000000000000:role/PolarisVendedRole"
polarisSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test signing key
)
type TestEnvironment struct {
seaweedDir string
weedBinary string
dataDir string
bindIP string
s3Port int
s3GrpcPort int
masterPort int
masterGrpcPort int
filerPort int
filerGrpcPort int
volumePort int
volumeGrpcPort int
polarisPort int
polarisAdminPort int
weedProcess *exec.Cmd
weedCancel context.CancelFunc
polarisContainer string
accessKey string
secretKey string
}
func NewTestEnvironment(t *testing.T) *TestEnvironment {
t.Helper()
if !testutil.HasDocker() {
t.Skip("Docker is required for Polaris integration tests")
}
wd, err := os.Getwd()
if err != nil {
t.Fatalf("Failed to get working directory: %v", err)
}
seaweedDir := wd
for i := 0; i < 6; i++ {
if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
break
}
seaweedDir = filepath.Dir(seaweedDir)
}
weedBinary := filepath.Join(seaweedDir, "weed", "weed")
if _, err := os.Stat(weedBinary); err != nil {
weedBinary = "weed"
if _, err := exec.LookPath(weedBinary); err != nil {
t.Skip("weed binary not found, skipping integration test")
}
}
dataDir, err := os.MkdirTemp("", "seaweed-polaris-test-*")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
bindIP := testutil.FindBindIP()
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
return &TestEnvironment{
seaweedDir: seaweedDir,
weedBinary: weedBinary,
dataDir: dataDir,
bindIP: bindIP,
s3Port: s3Port,
s3GrpcPort: s3GrpcPort,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
filerPort: filerPort,
filerGrpcPort: filerGrpcPort,
volumePort: volumePort,
volumeGrpcPort: volumeGrpcPort,
polarisPort: polarisPort,
polarisAdminPort: polarisAdminPort,
accessKey: "admin",
secretKey: "admin",
}
}
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Helper()
iamConfigPath := filepath.Join(env.dataDir, "iam.json")
iamConfig := fmt.Sprintf(`{
"identities": [
{
"name": "admin",
"credentials": [
{
"accessKey": "%s",
"secretKey": "%s"
}
],
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
}
],
"sts": {
"tokenDuration": "12h",
"maxSessionLength": "24h",
"issuer": "seaweedfs-sts",
"signingKey": "%s"
},
"roles": [
{
"roleName": "PolarisVendedRole",
"roleArn": "%s",
"trustPolicy": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": "*",
"Action": "sts:AssumeRole"
}
]
},
"attachedPolicies": ["FullAccess"]
}
],
"policies": [
{
"name": "FullAccess",
"document": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "*",
"Resource": "*"
}
]
}
}
]
}`, env.accessKey, env.secretKey, polarisSigningKey, polarisRoleArn)
if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
t.Fatalf("Failed to create IAM config: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
env.weedCancel = cancel
cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini",
"-master.port", fmt.Sprintf("%d", env.masterPort),
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
"-volume.port", fmt.Sprintf("%d", env.volumePort),
"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
"-filer.port", fmt.Sprintf("%d", env.filerPort),
"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
"-s3.port", fmt.Sprintf("%d", env.s3Port),
"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
"-s3.config", iamConfigPath,
"-s3.iam.config", iamConfigPath,
"-s3.iam.readOnly=false",
"-ip", env.bindIP,
"-ip.bind", "0.0.0.0",
"-dir", env.dataDir,
)
cmd.Dir = env.dataDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
t.Fatalf("Failed to start SeaweedFS: %v", err)
}
env.weedProcess = cmd
if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
t.Fatalf("S3 API failed to become ready")
}
}
func (env *TestEnvironment) StartPolaris(t *testing.T) {
t.Helper()
containerName := fmt.Sprintf("seaweed-polaris-%d", time.Now().UnixNano())
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
cmd := exec.CommandContext(ctx, "docker", "run", "-d", "--rm",
"--name", containerName,
"--add-host", "host.docker.internal:host-gateway",
"-p", fmt.Sprintf("%d:8181", env.polarisPort),
"-p", fmt.Sprintf("%d:8182", env.polarisAdminPort),
"-e", fmt.Sprintf("AWS_REGION=%s", polarisRegion),
"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", env.accessKey),
"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", env.secretKey),
"-e", fmt.Sprintf("POLARIS_BOOTSTRAP_CREDENTIALS=%s,%s,%s", polarisRealm, polarisRootClientID, polarisRootClientSecret),
"-e", fmt.Sprintf("polaris.realm-context.realms=%s", polarisRealm),
"-e", "quarkus.otel.sdk.disabled=true",
polarisImage,
)
output, err := cmd.CombinedOutput()
if err != nil {
if ctx.Err() == context.DeadlineExceeded {
t.Fatalf("Timed out waiting for Polaris container: %v\nOutput:\n%s", ctx.Err(), string(output))
}
t.Fatalf("Failed to start Polaris: %v\nOutput:\n%s", err, string(output))
}
env.polarisContainer = containerName
if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/q/health", env.polarisAdminPort), 60*time.Second) {
logCtx, logCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer logCancel()
logs, _ := exec.CommandContext(logCtx, "docker", "logs", env.polarisContainer).CombinedOutput()
t.Fatalf("Polaris failed to become ready\nLogs:\n%s", string(logs))
}
}
func (env *TestEnvironment) Cleanup(t *testing.T) {
t.Helper()
if env.weedCancel != nil {
env.weedCancel()
}
if env.weedProcess != nil {
time.Sleep(1 * time.Second)
_ = env.weedProcess.Wait()
}
if env.polarisContainer != "" {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_ = exec.CommandContext(ctx, "docker", "rm", "-f", env.polarisContainer).Run()
}
if env.dataDir != "" {
_ = os.RemoveAll(env.dataDir)
}
}
func (env *TestEnvironment) polarisEndpoint() string {
return fmt.Sprintf("http://127.0.0.1:%d", env.polarisPort)
}
func (env *TestEnvironment) s3Endpoint() string {
return fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
}
func (env *TestEnvironment) s3InternalEndpoint() string {
return fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
}
type polarisHTTPClient struct {
baseURL string
realm string
token string
httpClient *http.Client
}
func newPolarisHTTPClient(baseURL, realm, token string) *polarisHTTPClient {
return &polarisHTTPClient{
baseURL: baseURL,
realm: realm,
token: token,
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
func (c *polarisHTTPClient) doJSON(ctx context.Context, method, path string, body interface{}, out interface{}) error {
return c.doJSONWithHeaders(ctx, method, path, body, out, nil)
}
func (c *polarisHTTPClient) doJSONWithHeaders(ctx context.Context, method, path string, body interface{}, out interface{}, headers map[string]string) error {
var reader io.Reader
if body != nil {
encoded, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("encode request body: %w", err)
}
reader = bytes.NewReader(encoded)
}
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)
if err != nil {
return fmt.Errorf("create request: %w", err)
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
req.Header.Set("Accept", "application/json")
if c.realm != "" {
req.Header.Set("Polaris-Realm", c.realm)
}
if c.token != "" {
req.Header.Set("Authorization", "Bearer "+c.token)
}
for key, value := range headers {
req.Header.Set(key, value)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
bodyBytes, readErr := io.ReadAll(resp.Body)
if readErr != nil {
return fmt.Errorf("request failed with status %d and could not read response body: %w", resp.StatusCode, readErr)
}
return fmt.Errorf("request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
}
if out == nil {
return nil
}
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
return fmt.Errorf("decode response: %w", err)
}
return nil
}
func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials, pathStyle bool) (*s3.Client, error) {
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID {
return aws.Endpoint{
URL: endpoint,
SigningRegion: region,
HostnameImmutable: true,
}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
config.WithEndpointResolverWithOptions(resolver),
)
if err != nil {
return nil, err
}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = pathStyle
}), nil
}
-764
View File
@@ -1,764 +0,0 @@
package polaris
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type polarisSession struct {
catalogName string
bucketName string
token string
baseLocation string
}
type polarisTableSetup struct {
namespace string
table string
dataKeyPrefix string
s3Client *s3.Client
}
type polarisCatalogClient struct {
http *polarisHTTPClient
catalog string
}
type polarisCredentials struct {
ClientID string `json:"clientId"`
ClientSecret string `json:"clientSecret"`
}
type createPrincipalResponse struct {
Credentials polarisCredentials `json:"credentials"`
}
type createCatalogRequest struct {
Catalog polarisCatalog `json:"catalog"`
}
type polarisCatalog struct {
Name string `json:"name"`
Type string `json:"type"`
ReadOnly bool `json:"readOnly"`
Properties map[string]string `json:"properties"`
StorageConfigInfo polarisStorageConfig `json:"storageConfigInfo"`
}
type polarisStorageConfig struct {
StorageType string `json:"storageType"`
AllowedLocations []string `json:"allowedLocations"`
Endpoint string `json:"endpoint"`
EndpointInternal string `json:"endpointInternal"`
StsEndpoint string `json:"stsEndpoint"`
PathStyleAccess bool `json:"pathStyleAccess"`
RoleArn string `json:"roleArn"`
Region string `json:"region"`
}
type createNamespaceRequest struct {
Namespace []string `json:"namespace"`
}
type createTableRequest struct {
Name string `json:"name"`
Location string `json:"location"`
Schema icebergSchema `json:"schema"`
PartitionSpec icebergPartition `json:"partition-spec"`
SortOrder icebergSortOrder `json:"sort-order"`
Properties map[string]string `json:"properties"`
}
type icebergSchema struct {
Type string `json:"type"`
SchemaID int `json:"schema-id"`
Fields []icebergSchemaField `json:"fields"`
}
type icebergSchemaField struct {
ID int `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Required bool `json:"required"`
}
type icebergPartition struct {
SpecID int `json:"spec-id"`
Fields []icebergPartitionField `json:"fields"`
}
type icebergPartitionField struct {
SourceID int `json:"source-id"`
FieldID int `json:"field-id"`
Name string `json:"name"`
Transform string `json:"transform"`
}
type icebergSortOrder struct {
OrderID int `json:"order-id"`
Fields []icebergSortField `json:"fields"`
}
type icebergSortField struct {
SourceID int `json:"source-id"`
Direction string `json:"direction"`
NullOrder string `json:"null-order"`
}
type loadTableResponse struct {
Config map[string]string `json:"config"`
StorageCredentials []storageCredential `json:"storage-credentials"`
}
type loadCredentialsResponse struct {
StorageCredentials []storageCredential `json:"storage-credentials"`
}
type storageCredential struct {
Prefix string `json:"prefix"`
Config map[string]string `json:"config"`
}
func bootstrapPolarisTest(t *testing.T, env *TestEnvironment) (context.Context, context.CancelFunc, polarisSession, *polarisTableSetup, func()) {
t.Helper()
t.Logf(">>> Starting SeaweedFS with Polaris configuration...")
env.StartSeaweedFS(t)
t.Logf(">>> SeaweedFS started.")
t.Logf(">>> Starting Polaris...")
env.StartPolaris(t)
t.Logf(">>> Polaris started.")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
session := newPolarisSession(t, ctx, env)
setup, cleanup := setupPolarisTable(t, ctx, env, session)
return ctx, cancel, session, setup, cleanup
}
func TestPolarisIntegration(t *testing.T) {
env := NewTestEnvironment(t)
defer env.Cleanup(t)
ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
defer cancel()
defer cleanup()
objectKey := fmt.Sprintf("%s/hello-%d.txt", setup.dataKeyPrefix, time.Now().UnixNano())
payload := []byte("polaris")
if _, err := setup.s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader(payload),
}); err != nil {
t.Fatalf("PutObject failed: %v", err)
}
listObjects, err := setup.s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(session.bucketName),
Prefix: aws.String(objectKey),
})
if err != nil {
t.Fatalf("ListObjectsV2 failed: %v", err)
}
found := false
for _, obj := range listObjects.Contents {
if aws.ToString(obj.Key) == objectKey {
found = true
break
}
}
if !found {
t.Fatalf("Object %s not found in list", objectKey)
}
getResp, err := setup.s3Client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("GetObject failed: %v", err)
}
body, err := io.ReadAll(getResp.Body)
_ = getResp.Body.Close()
if err != nil {
t.Fatalf("Read object body failed: %v", err)
}
if !bytes.Equal(body, payload) {
t.Fatalf("Unexpected object payload: got %q want %q", string(body), string(payload))
}
if _, err := setup.s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
}); err != nil {
t.Fatalf("DeleteObject failed: %v", err)
}
}
func TestPolarisTableIntegration(t *testing.T) {
env := NewTestEnvironment(t)
defer env.Cleanup(t)
ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
defer cancel()
defer cleanup()
objectKey := fmt.Sprintf("%s/part-%d.parquet", setup.dataKeyPrefix, time.Now().UnixNano())
createResp, err := setup.s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("CreateMultipartUpload failed: %v", err)
}
uploadID := aws.ToString(createResp.UploadId)
multipartCompleted := false
defer func() {
if uploadID == "" || multipartCompleted {
return
}
_, _ = setup.s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
})
}()
partSize := 5 * 1024 * 1024
part1 := bytes.Repeat([]byte("a"), partSize)
part2 := bytes.Repeat([]byte("b"), 1024*1024)
part1Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(1),
Body: bytes.NewReader(part1),
})
if err != nil {
t.Fatalf("UploadPart 1 failed: %v", err)
}
part2Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(2),
Body: bytes.NewReader(part2),
})
if err != nil {
t.Fatalf("UploadPart 2 failed: %v", err)
}
_, err = setup.s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
MultipartUpload: &s3types.CompletedMultipartUpload{
Parts: []s3types.CompletedPart{
{
ETag: part1Resp.ETag,
PartNumber: aws.Int32(1),
},
{
ETag: part2Resp.ETag,
PartNumber: aws.Int32(2),
},
},
},
})
if err != nil {
t.Fatalf("CompleteMultipartUpload failed: %v", err)
}
multipartCompleted = true
headResp, err := setup.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("HeadObject after multipart upload failed: %v", err)
}
expectedSize := int64(len(part1) + len(part2))
if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
}
}
func newPolarisSession(t *testing.T, ctx context.Context, env *TestEnvironment) polarisSession {
t.Helper()
adminCreds := aws.Credentials{
AccessKeyID: env.accessKey,
SecretAccessKey: env.secretKey,
Source: "polaris-admin",
}
adminS3, err := newS3Client(ctx, env.s3Endpoint(), polarisRegion, adminCreds, true)
if err != nil {
t.Fatalf("Create admin S3 client failed: %v", err)
}
bucketName := fmt.Sprintf("polaris-bucket-%d", time.Now().UnixNano())
if _, err := adminS3.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
}); err != nil {
t.Fatalf("CreateBucket failed: %v", err)
}
policyDoc := map[string]interface{}{
"Version": "2012-10-17",
"Statement": []map[string]interface{}{
{
"Sid": "AllowPolarisVendedAccess",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:*",
"Resource": []string{
fmt.Sprintf("arn:aws:s3:::%s", bucketName),
fmt.Sprintf("arn:aws:s3:::%s/polaris/*", bucketName),
},
},
},
}
policyBytes, err := json.Marshal(policyDoc)
if err != nil {
t.Fatalf("Failed to marshal bucket policy: %v", err)
}
if _, err := adminS3.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
Bucket: aws.String(bucketName),
Policy: aws.String(string(policyBytes)),
}); err != nil {
t.Fatalf("PutBucketPolicy failed: %v", err)
}
rootToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), polarisRootClientID, polarisRootClientSecret)
if err != nil {
t.Fatalf("Polaris root token request failed: %v", err)
}
managementClient := newPolarisHTTPClient(env.polarisEndpoint(), polarisRealm, rootToken)
catalogName := fmt.Sprintf("polaris_catalog_%d", time.Now().UnixNano())
baseLocation := fmt.Sprintf("s3://%s/polaris", bucketName)
catalogRequest := createCatalogRequest{
Catalog: polarisCatalog{
Name: catalogName,
Type: "INTERNAL",
ReadOnly: false,
Properties: map[string]string{
"default-base-location": baseLocation,
},
StorageConfigInfo: polarisStorageConfig{
StorageType: "S3",
AllowedLocations: []string{baseLocation},
Endpoint: env.s3Endpoint(),
EndpointInternal: env.s3InternalEndpoint(),
StsEndpoint: env.s3InternalEndpoint(),
PathStyleAccess: true,
RoleArn: polarisRoleArn,
Region: polarisRegion,
},
},
}
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/catalogs", catalogRequest, nil); err != nil {
t.Fatalf("Create catalog failed: %v", err)
}
principalName := fmt.Sprintf("polaris_user_%d", time.Now().UnixNano())
principalRoleName := fmt.Sprintf("polaris_principal_role_%d", time.Now().UnixNano())
catalogRoleName := fmt.Sprintf("polaris_catalog_role_%d", time.Now().UnixNano())
var principalResp createPrincipalResponse
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principals", map[string]interface{}{
"principal": map[string]interface{}{
"name": principalName,
"properties": map[string]string{},
},
}, &principalResp); err != nil {
t.Fatalf("Create principal failed: %v", err)
}
if principalResp.Credentials.ClientID == "" || principalResp.Credentials.ClientSecret == "" {
t.Fatalf("Missing principal credentials in response")
}
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principal-roles", map[string]interface{}{
"principalRole": map[string]interface{}{
"name": principalRoleName,
"properties": map[string]string{},
},
}, nil); err != nil {
t.Fatalf("Create principal role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPost, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles", url.PathEscape(catalogName)), map[string]interface{}{
"catalogRole": map[string]interface{}{
"name": catalogRoleName,
"properties": map[string]string{},
},
}, nil); err != nil {
t.Fatalf("Create catalog role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principals/%s/principal-roles", url.PathEscape(principalName)), map[string]interface{}{
"principalRole": map[string]interface{}{
"name": principalRoleName,
},
}, nil); err != nil {
t.Fatalf("Assign principal role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principal-roles/%s/catalog-roles/%s", url.PathEscape(principalRoleName), url.PathEscape(catalogName)), map[string]interface{}{
"catalogRole": map[string]interface{}{
"name": catalogRoleName,
},
}, nil); err != nil {
t.Fatalf("Assign catalog role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles/%s/grants", url.PathEscape(catalogName), url.PathEscape(catalogRoleName)), map[string]interface{}{
"type": "catalog",
"privilege": "CATALOG_MANAGE_CONTENT",
}, nil); err != nil {
t.Fatalf("Grant catalog privilege failed: %v", err)
}
userToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), principalResp.Credentials.ClientID, principalResp.Credentials.ClientSecret)
if err != nil {
t.Fatalf("Polaris user token request failed: %v", err)
}
return polarisSession{
catalogName: catalogName,
bucketName: bucketName,
token: userToken,
baseLocation: baseLocation,
}
}
func setupPolarisTable(t *testing.T, ctx context.Context, env *TestEnvironment, session polarisSession) (*polarisTableSetup, func()) {
t.Helper()
catalogClient := newPolarisCatalogClient(env.polarisEndpoint(), polarisRealm, session.token, session.catalogName)
namespace := fmt.Sprintf("polaris_ns_%d", time.Now().UnixNano())
table := fmt.Sprintf("polaris_table_%d", time.Now().UnixNano())
if err := catalogClient.CreateNamespace(ctx, namespace); err != nil {
t.Fatalf("CreateNamespace failed: %v", err)
}
location := fmt.Sprintf("%s/%s/%s", session.baseLocation, namespace, table)
if err := catalogClient.CreateTable(ctx, namespace, table, location); err != nil {
t.Fatalf("CreateTable failed: %v", err)
}
loadResp, err := catalogClient.LoadTable(ctx, namespace, table)
if err != nil {
t.Fatalf("LoadTable failed: %v", err)
}
credsResp, err := catalogClient.LoadCredentials(ctx, namespace, table)
if err != nil {
t.Fatalf("LoadCredentials failed: %v", err)
}
if len(credsResp.StorageCredentials) == 0 {
t.Fatalf("LoadCredentials returned no storage credentials")
}
credentialSource := &loadTableResponse{
Config: loadResp.Config,
StorageCredentials: credsResp.StorageCredentials,
}
dataPrefix := location + "/data"
creds, endpoint, region, pathStyle, err := extractS3Credentials(credentialSource, dataPrefix, env.s3Endpoint(), polarisRegion)
if err != nil {
t.Fatalf("Extract vended credentials failed: %v", err)
}
dataKeyPrefix, err := s3URIToKeyPrefix(dataPrefix, session.bucketName)
if err != nil {
t.Fatalf("Invalid data prefix %s: %v", dataPrefix, err)
}
s3Client, err := newS3Client(ctx, endpoint, region, creds, pathStyle)
if err != nil {
t.Fatalf("Create vended S3 client failed: %v", err)
}
cleanup := func() {
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cleanupCancel()
if err := catalogClient.DeleteTable(cleanupCtx, namespace, table); err != nil {
t.Logf("DeleteTable failed: %v", err)
}
if err := catalogClient.DeleteNamespace(cleanupCtx, namespace); err != nil {
t.Logf("DeleteNamespace failed: %v", err)
}
}
return &polarisTableSetup{
namespace: namespace,
table: table,
dataKeyPrefix: dataKeyPrefix,
s3Client: s3Client,
}, cleanup
}
func fetchPolarisToken(ctx context.Context, baseURL, clientID, clientSecret string) (string, error) {
form := url.Values{}
form.Set("grant_type", "client_credentials")
form.Set("client_id", clientID)
form.Set("client_secret", clientSecret)
form.Set("scope", "PRINCIPAL_ROLE:ALL")
req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/api/catalog/v1/oauth/tokens", strings.NewReader(form.Encode()))
if err != nil {
return "", fmt.Errorf("create token request: %w", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("token request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, readErr := io.ReadAll(resp.Body)
if readErr != nil {
return "", fmt.Errorf("token request failed with status %d and reading body: %w", resp.StatusCode, readErr)
}
return "", fmt.Errorf("token request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
var tokenResp struct {
AccessToken string `json:"access_token"`
}
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
return "", fmt.Errorf("decode token response: %w", err)
}
if tokenResp.AccessToken == "" {
return "", fmt.Errorf("missing access token in response")
}
return tokenResp.AccessToken, nil
}
func newPolarisCatalogClient(baseURL, realm, token, catalog string) *polarisCatalogClient {
return &polarisCatalogClient{
http: newPolarisHTTPClient(baseURL, realm, token),
catalog: catalog,
}
}
func (c *polarisCatalogClient) CreateNamespace(ctx context.Context, namespace string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces", url.PathEscape(c.catalog))
req := createNamespaceRequest{Namespace: []string{namespace}}
return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
}
func (c *polarisCatalogClient) DeleteNamespace(ctx context.Context, namespace string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s", url.PathEscape(c.catalog), url.PathEscape(namespace))
return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
}
func (c *polarisCatalogClient) CreateTable(ctx context.Context, namespace, table, location string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables", url.PathEscape(c.catalog), url.PathEscape(namespace))
req := createTableRequest{
Name: table,
Location: location,
Schema: icebergSchema{
Type: "struct",
SchemaID: 0,
Fields: []icebergSchemaField{
{
ID: 1,
Name: "id",
Type: "long",
Required: false,
},
},
},
PartitionSpec: icebergPartition{
SpecID: 0,
Fields: []icebergPartitionField{},
},
SortOrder: icebergSortOrder{
OrderID: 0,
Fields: []icebergSortField{},
},
Properties: map[string]string{
"format-version": "2",
},
}
return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
}
func (c *polarisCatalogClient) DeleteTable(ctx context.Context, namespace, table string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
}
func (c *polarisCatalogClient) LoadTable(ctx context.Context, namespace, table string) (*loadTableResponse, error) {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
var resp loadTableResponse
headers := map[string]string{
"X-Iceberg-Access-Delegation": "vended-credentials",
}
if err := c.http.doJSONWithHeaders(ctx, http.MethodGet, path, nil, &resp, headers); err != nil {
return nil, err
}
return &resp, nil
}
func (c *polarisCatalogClient) LoadCredentials(ctx context.Context, namespace, table string) (*loadCredentialsResponse, error) {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s/credentials", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
var resp loadCredentialsResponse
if err := c.http.doJSON(ctx, http.MethodGet, path, nil, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoint, fallbackRegion string) (aws.Credentials, string, string, bool, error) {
credentialConfig, err := selectStorageConfig(load, targetPrefix)
if err != nil {
return aws.Credentials{}, "", "", false, err
}
lookupConfig := func(key string) string {
if load != nil && load.Config != nil {
if val, ok := load.Config[key]; ok && strings.TrimSpace(val) != "" {
return strings.TrimSpace(val)
}
}
if val, ok := credentialConfig[key]; ok && strings.TrimSpace(val) != "" {
return strings.TrimSpace(val)
}
return ""
}
accessKey := strings.TrimSpace(credentialConfig["s3.access-key-id"])
secretKey := strings.TrimSpace(credentialConfig["s3.secret-access-key"])
sessionToken := strings.TrimSpace(credentialConfig["s3.session-token"])
if accessKey == "" || secretKey == "" {
return aws.Credentials{}, "", "", false, fmt.Errorf("missing s3.access-key-id or s3.secret-access-key in selected storage credential")
}
endpoint := lookupConfig("s3.endpoint")
if endpoint == "" {
endpoint = fallbackEndpoint
}
if endpoint != "" && !strings.HasPrefix(endpoint, "http") {
endpoint = "http://" + endpoint
}
region := lookupConfig("client.region")
if region == "" {
region = fallbackRegion
}
pathStyle := true
if value := lookupConfig("s3.path-style-access"); value != "" {
pathStyle = strings.EqualFold(value, "true")
}
return aws.Credentials{
AccessKeyID: accessKey,
SecretAccessKey: secretKey,
SessionToken: sessionToken,
Source: "polaris-vended",
}, endpoint, region, pathStyle, nil
}
func selectStorageConfig(load *loadTableResponse, targetPrefix string) (map[string]string, error) {
if load == nil {
return nil, fmt.Errorf("load table response is nil")
}
switch len(load.StorageCredentials) {
case 0:
if load.Config == nil {
return nil, fmt.Errorf("polaris returned no storage credentials or config")
}
return load.Config, nil
case 1:
cred := load.StorageCredentials[0]
if cred.Config == nil {
return nil, fmt.Errorf("storage credential for prefix %s returned nil config", cred.Prefix)
}
return cred.Config, nil
default:
if targetPrefix == "" {
return nil, fmt.Errorf("multiple storage credentials (%d) returned but no target prefix provided", len(load.StorageCredentials))
}
normalizedTarget := normalizePrefix(targetPrefix)
if normalizedTarget == "" {
return nil, fmt.Errorf("target prefix %q normalized to empty string", targetPrefix)
}
var bestConfig map[string]string
bestLen := -1
for _, cred := range load.StorageCredentials {
if cred.Config == nil {
continue
}
prefix := normalizePrefix(cred.Prefix)
if prefix == "" {
if bestLen < 0 {
bestLen = 0
bestConfig = cred.Config
}
continue
}
if normalizedTarget == prefix || strings.HasPrefix(normalizedTarget, prefix+"/") {
if len(prefix) > bestLen {
bestLen = len(prefix)
bestConfig = cred.Config
}
}
}
if bestConfig != nil {
return bestConfig, nil
}
return nil, fmt.Errorf("none of the %d storage credentials matched prefix %s", len(load.StorageCredentials), targetPrefix)
}
}
func normalizePrefix(prefix string) string {
p := strings.TrimSpace(prefix)
p = strings.TrimSuffix(p, "/")
return p
}
func s3URIToKeyPrefix(uri, bucket string) (string, error) {
prefix := "s3://" + bucket + "/"
if !strings.HasPrefix(uri, prefix) {
return "", fmt.Errorf("uri %q does not match bucket %q", uri, bucket)
}
keyPrefix := strings.TrimPrefix(uri, prefix)
keyPrefix = strings.TrimPrefix(keyPrefix, "/")
if keyPrefix == "" {
return "", fmt.Errorf("empty key prefix in uri %q", uri)
}
return keyPrefix, nil
}
+7 -4
View File
@@ -23,6 +23,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
if err != nil {
t.Fatalf("create status request: %v", err)
}
statusReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-1")
statusResp := framework.DoRequest(t, client, statusReq)
statusBody := framework.ReadAllAndClose(t, statusResp)
@@ -32,8 +34,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
if got := statusResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
t.Fatalf("expected /status Server header to contain SeaweedFS Volume, got %q", got)
}
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
t.Fatal("expected server-generated request id in response header")
if got := statusResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-1" {
t.Fatalf("expected echoed request id, got %q", got)
}
var payload map[string]interface{}
@@ -47,6 +49,7 @@ func TestAdminStatusAndHealthz(t *testing.T) {
}
healthReq := mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/healthz")
healthReq.Header.Set(request_id.AmzRequestIDHeader, "test-request-id-2")
healthResp := framework.DoRequest(t, client, healthReq)
_ = framework.ReadAllAndClose(t, healthResp)
if healthResp.StatusCode != http.StatusOK {
@@ -55,8 +58,8 @@ func TestAdminStatusAndHealthz(t *testing.T) {
if got := healthResp.Header.Get("Server"); !strings.Contains(got, "SeaweedFS Volume") {
t.Fatalf("expected /healthz Server header to contain SeaweedFS Volume, got %q", got)
}
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got == "" {
t.Fatal("expected /healthz server-generated request id in response header")
if got := healthResp.Header.Get(request_id.AmzRequestIDHeader); got != "test-request-id-2" {
t.Fatalf("expected /healthz echoed request id, got %q", got)
}
uiResp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
-149
View File
@@ -1,149 +0,0 @@
package dash
import (
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
)
const (
adminLockName = cluster.AdminShellLockName
adminLockClientName = "admin-plugin"
)
// AdminLockManager coordinates exclusive admin locks with reference counting.
// It is safe for concurrent use.
type AdminLockManager struct {
locker *exclusive_locks.ExclusiveLocker
clientName string
mu sync.Mutex
cond *sync.Cond
acquiring bool
holdCount int
lastAcquiredAt time.Time
lastReleasedAt time.Time
waitingSince time.Time
waitingReason string
currentReason string
}
func NewAdminLockManager(masterClient *wdclient.MasterClient, clientName string) *AdminLockManager {
if masterClient == nil {
return nil
}
if clientName == "" {
clientName = adminLockClientName
}
manager := &AdminLockManager{
locker: exclusive_locks.NewExclusiveLocker(masterClient, adminLockName),
clientName: clientName,
}
manager.cond = sync.NewCond(&manager.mu)
return manager
}
func (m *AdminLockManager) Acquire(reason string) (func(), error) {
if m == nil || m.locker == nil {
return func() {}, nil
}
m.mu.Lock()
if reason != "" {
m.locker.SetMessage(reason)
m.currentReason = reason
}
for m.acquiring {
m.cond.Wait()
}
if m.holdCount == 0 {
m.acquiring = true
m.waitingSince = time.Now().UTC()
m.waitingReason = reason
m.mu.Unlock()
m.locker.RequestLock(m.clientName)
m.mu.Lock()
m.acquiring = false
m.holdCount = 1
m.lastAcquiredAt = time.Now().UTC()
m.waitingSince = time.Time{}
m.waitingReason = ""
m.cond.Broadcast()
m.mu.Unlock()
return m.Release, nil
}
m.holdCount++
if reason != "" {
m.currentReason = reason
}
m.mu.Unlock()
return m.Release, nil
}
func (m *AdminLockManager) Release() {
if m == nil || m.locker == nil {
return
}
m.mu.Lock()
if m.holdCount <= 0 {
m.mu.Unlock()
return
}
m.holdCount--
shouldRelease := m.holdCount == 0
m.mu.Unlock()
if shouldRelease {
m.mu.Lock()
m.lastReleasedAt = time.Now().UTC()
m.currentReason = ""
m.mu.Unlock()
m.locker.ReleaseLock()
}
}
type LockStatus struct {
Held bool `json:"held"`
HoldCount int `json:"hold_count"`
Acquiring bool `json:"acquiring"`
Message string `json:"message,omitempty"`
WaitingReason string `json:"waiting_reason,omitempty"`
LastAcquiredAt *time.Time `json:"last_acquired_at,omitempty"`
LastReleasedAt *time.Time `json:"last_released_at,omitempty"`
WaitingSince *time.Time `json:"waiting_since,omitempty"`
}
func (m *AdminLockManager) Status() LockStatus {
if m == nil {
return LockStatus{}
}
m.mu.Lock()
defer m.mu.Unlock()
status := LockStatus{
Held: m.holdCount > 0,
HoldCount: m.holdCount,
Acquiring: m.acquiring,
Message: m.currentReason,
WaitingReason: m.waitingReason,
}
if !m.lastAcquiredAt.IsZero() {
at := m.lastAcquiredAt
status.LastAcquiredAt = &at
}
if !m.lastReleasedAt.IsZero() {
at := m.lastReleasedAt
status.LastReleasedAt = &at
}
if !m.waitingSince.IsZero() {
at := m.waitingSince
status.WaitingSince = &at
}
return status
}
-61
View File
@@ -1,61 +0,0 @@
package dash
import (
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
)
const adminPresenceClientName = "admin-server"
type adminPresenceLock struct {
locker *exclusive_locks.ExclusiveLocker
stopCh chan struct{}
}
func newAdminPresenceLock(masterClient *wdclient.MasterClient) *adminPresenceLock {
if masterClient == nil {
return nil
}
return &adminPresenceLock{
locker: exclusive_locks.NewExclusiveLocker(masterClient, cluster.AdminServerPresenceLockName),
stopCh: make(chan struct{}),
}
}
func (l *adminPresenceLock) Start() {
if l == nil || l.locker == nil {
return
}
l.locker.SetMessage("admin server connected")
go func() {
ticker := time.NewTicker(5 * time.Second)
defer ticker.Stop()
for {
if !l.locker.IsLocked() {
l.locker.RequestLock(adminPresenceClientName)
}
select {
case <-l.stopCh:
return
case <-ticker.C:
}
}
}()
}
func (l *adminPresenceLock) Stop() {
if l == nil {
return
}
select {
case <-l.stopCh:
default:
close(l.stopCh)
}
if l.locker != nil {
l.locker.ReleaseLock()
}
}
+11 -156
View File
@@ -11,7 +11,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
adminplugin "github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/cluster"
clustermaintenance "github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -99,9 +98,6 @@ type AdminServer struct {
// Maintenance system
maintenanceManager *maintenance.MaintenanceManager
plugin *adminplugin.Plugin
pluginLock *AdminLockManager
adminPresenceLock *adminPresenceLock
expireJobHandler func(jobID string, reason string) (*adminplugin.TrackedJob, bool, error)
// Topic retention purger
topicRetentionPurger *TopicRetentionPurger
@@ -138,12 +134,6 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
ctx := context.Background()
go masterClient.KeepConnectedToMaster(ctx)
lockManager := NewAdminLockManager(masterClient, adminLockClientName)
presenceLock := newAdminPresenceLock(masterClient)
if presenceLock != nil {
presenceLock.Start()
}
server := &AdminServer{
masterClient: masterClient,
templateFS: templateFS,
@@ -155,8 +145,6 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
collectionStatsCacheThreshold: defaultStatsCacheTimeout,
s3TablesManager: newS3TablesManager(),
icebergPort: icebergPort,
pluginLock: lockManager,
adminPresenceLock: presenceLock,
}
// Initialize topic retention purger
@@ -235,19 +223,20 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
}()
}
pluginOpts := adminplugin.Options{
plugin, err := adminplugin.New(adminplugin.Options{
DataDir: dataDir,
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
return server.buildDefaultPluginClusterContext(), nil
},
LockManager: lockManager,
ConfigDefaultsProvider: server.enrichConfigDefaults,
}
plugin, err := adminplugin.New(pluginOpts)
})
if err != nil && dataDir != "" {
glog.Warningf("Failed to initialize plugin with dataDir=%q: %v. Falling back to in-memory plugin state.", dataDir, err)
pluginOpts.DataDir = ""
plugin, err = adminplugin.New(pluginOpts)
plugin, err = adminplugin.New(adminplugin.Options{
DataDir: "",
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
return server.buildDefaultPluginClusterContext(), nil
},
})
}
if err != nil {
glog.Errorf("Failed to initialize plugin: %v", err)
@@ -271,89 +260,6 @@ func (s *AdminServer) loadTaskConfigurationsFromPersistence() {
configUpdateRegistry.UpdateAllConfigs(s.configPersistence)
}
// enrichConfigDefaults is called by the plugin when bootstrapping a job type's
// default config from its descriptor. For admin_script, it fetches maintenance
// scripts from the master and uses them as the script default.
//
// MIGRATION: This exists to help users migrate from master.toml [master.maintenance]
// to the admin script plugin worker. Remove after March 2027.
func (s *AdminServer) enrichConfigDefaults(cfg *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig {
if cfg.JobType != "admin_script" {
return cfg
}
var maintenanceScripts string
var sleepMinutes uint32
err := s.WithMasterClient(func(client master_pb.SeaweedClient) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
resp, err := client.GetMasterConfiguration(ctx, &master_pb.GetMasterConfigurationRequest{})
if err != nil {
return err
}
maintenanceScripts = resp.MaintenanceScripts
sleepMinutes = resp.MaintenanceSleepMinutes
return nil
})
if err != nil {
glog.V(1).Infof("Could not fetch master configuration for admin_script defaults: %v", err)
return cfg
}
script := cleanMaintenanceScript(maintenanceScripts)
if script == "" {
return cfg
}
interval := int64(sleepMinutes)
if interval <= 0 {
interval = clustermaintenance.DefaultMaintenanceSleepMinutes
}
glog.V(0).Infof("Enriching admin_script defaults from master maintenance scripts (interval=%dm)", interval)
if cfg.AdminConfigValues == nil {
cfg.AdminConfigValues = make(map[string]*plugin_pb.ConfigValue)
}
cfg.AdminConfigValues["script"] = &plugin_pb.ConfigValue{
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: script},
}
cfg.AdminConfigValues["run_interval_minutes"] = &plugin_pb.ConfigValue{
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: interval},
}
cfg.UpdatedBy = "master_migration"
return cfg
}
// cleanMaintenanceScript strips lock/unlock commands and normalizes a
// maintenance script string for use with the admin script plugin worker.
//
// MIGRATION: Used by enrichConfigDefaults. Remove after March 2027.
func cleanMaintenanceScript(script string) string {
script = strings.ReplaceAll(script, "\r\n", "\n")
var lines []string
for _, line := range strings.Split(script, "\n") {
trimmed := strings.TrimSpace(line)
if trimmed == "" || strings.HasPrefix(trimmed, "#") {
continue
}
// Strip inline comments (e.g., "lock # migration note")
if idx := strings.Index(trimmed, "#"); idx >= 0 {
trimmed = strings.TrimSpace(trimmed[:idx])
if trimmed == "" {
continue
}
}
firstToken := strings.ToLower(strings.Fields(trimmed)[0])
if firstToken == "lock" || firstToken == "unlock" {
continue
}
lines = append(lines, trimmed)
}
return strings.Join(lines, "\n")
}
// GetCredentialManager returns the credential manager
func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
return s.credentialManager
@@ -427,7 +333,7 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
return err
}
if resp.Entry != nil && resp.Entry.IsDirectory {
if resp.Entry.IsDirectory {
bucketName := resp.Entry.Name
if strings.HasPrefix(bucketName, ".") {
// Skip internal/system directories from Object Store bucket listing.
@@ -480,18 +386,13 @@ func (s *AdminServer) GetS3Buckets() ([]S3Bucket, error) {
}
}
var createdAt, lastModified time.Time
if resp.Entry.Attributes != nil {
createdAt = time.Unix(resp.Entry.Attributes.Crtime, 0)
lastModified = time.Unix(resp.Entry.Attributes.Mtime, 0)
}
bucket := S3Bucket{
Name: bucketName,
CreatedAt: createdAt,
CreatedAt: time.Unix(resp.Entry.Attributes.Crtime, 0),
LogicalSize: logicalSize,
PhysicalSize: physicalSize,
ObjectCount: objectCount,
LastModified: lastModified,
LastModified: time.Unix(resp.Entry.Attributes.Mtime, 0),
Quota: quota,
QuotaEnabled: quotaEnabled,
VersioningStatus: versioningStatus,
@@ -988,13 +889,6 @@ func (s *AdminServer) GetPlugin() *adminplugin.Plugin {
return s.plugin
}
func (s *AdminServer) acquirePluginLock(reason string) (func(), error) {
if s == nil || s.pluginLock == nil {
return func() {}, nil
}
return s.pluginLock.Acquire(reason)
}
// RequestPluginJobTypeDescriptor asks one worker for job type schema and returns the descriptor.
func (s *AdminServer) RequestPluginJobTypeDescriptor(ctx context.Context, jobType string, forceRefresh bool) (*plugin_pb.JobTypeDescriptor, error) {
if s.plugin == nil {
@@ -1037,13 +931,6 @@ func (s *AdminServer) RunPluginDetection(
if s.plugin == nil {
return nil, fmt.Errorf("plugin is not enabled")
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
if err != nil {
return nil, err
}
if releaseLock != nil {
defer releaseLock()
}
return s.plugin.RunDetection(ctx, jobType, clusterContext, maxResults)
}
@@ -1069,13 +956,6 @@ func (s *AdminServer) RunPluginDetectionWithReport(
if s.plugin == nil {
return nil, fmt.Errorf("plugin is not enabled")
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
if err != nil {
return nil, err
}
if releaseLock != nil {
defer releaseLock()
}
return s.plugin.RunDetectionWithReport(ctx, jobType, clusterContext, maxResults)
}
@@ -1089,17 +969,6 @@ func (s *AdminServer) ExecutePluginJob(
if s.plugin == nil {
return nil, fmt.Errorf("plugin is not enabled")
}
jobType := ""
if job != nil {
jobType = strings.TrimSpace(job.JobType)
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin execution %s", jobType))
if err != nil {
return nil, err
}
if releaseLock != nil {
defer releaseLock()
}
return s.plugin.ExecuteJob(ctx, job, clusterContext, attempt)
}
@@ -1151,17 +1020,6 @@ func (s *AdminServer) GetPluginJobDetail(jobID string, activityLimit, relatedLim
return s.plugin.BuildJobDetail(jobID, activityLimit, relatedLimit)
}
// ExpirePluginJob marks an active plugin job as failed so it no longer blocks scheduling.
func (s *AdminServer) ExpirePluginJob(jobID, reason string) (*adminplugin.TrackedJob, bool, error) {
if handler := s.expireJobHandler; handler != nil {
return handler(jobID, reason)
}
if s.plugin == nil {
return nil, false, fmt.Errorf("plugin is not enabled")
}
return s.plugin.ExpireJob(jobID, reason)
}
// ListPluginActivities returns plugin job activities for monitoring.
func (s *AdminServer) ListPluginActivities(jobType string, limit int) []adminplugin.JobActivity {
if s.plugin == nil {
@@ -1378,9 +1236,6 @@ func (s *AdminServer) Shutdown() {
// Stop maintenance manager
s.StopMaintenanceManager()
if s.adminPresenceLock != nil {
s.adminPresenceLock.Stop()
}
if s.plugin != nil {
s.plugin.Shutdown()
-77
View File
@@ -1,77 +0,0 @@
// MIGRATION: Tests for enrichConfigDefaults helpers. Remove after March 2027.
package dash
import "testing"
func TestCleanMaintenanceScript(t *testing.T) {
tests := []struct {
name string
input string
expected string
}{
{
name: "empty",
input: "",
expected: "",
},
{
name: "only lock unlock",
input: " lock\n unlock\n",
expected: "",
},
{
name: "strips lock and unlock",
input: " lock\n ec.balance -apply\n volume.fix.replication -apply\n unlock\n",
expected: "ec.balance -apply\nvolume.fix.replication -apply",
},
{
name: "case insensitive lock",
input: "Lock\nec.balance -apply\nUNLOCK",
expected: "ec.balance -apply",
},
{
name: "preserves comments removal",
input: "lock\n# a comment\nec.balance -apply\nunlock",
expected: "ec.balance -apply",
},
{
name: "no lock unlock present",
input: "ec.balance -apply\nvolume.fix.replication -apply",
expected: "ec.balance -apply\nvolume.fix.replication -apply",
},
{
name: "windows line endings",
input: "lock\r\nec.balance -apply\r\nunlock\r\n",
expected: "ec.balance -apply",
},
{
name: "lock with inline comment",
input: "lock # migration\nec.balance -apply\nunlock # done",
expected: "ec.balance -apply",
},
{
name: "command with inline comment preserved",
input: "lock\nec.balance -apply # rebalance shards\nunlock",
expected: "ec.balance -apply",
},
{
name: "only inline comment after stripping",
input: "# full line comment\n # indented comment\n",
expected: "",
},
{
name: "typical master default",
input: "\n lock\n ec.encode -fullPercent=95 -quietFor=1h\n ec.rebuild -apply\n ec.balance -apply\n fs.log.purge -daysAgo=7\n volume.deleteEmpty -quietFor=24h -apply\n volume.balance -apply\n volume.fix.replication -apply\n s3.clean.uploads -timeAgo=24h\n unlock\n",
expected: "ec.encode -fullPercent=95 -quietFor=1h\nec.rebuild -apply\nec.balance -apply\nfs.log.purge -daysAgo=7\nvolume.deleteEmpty -quietFor=24h -apply\nvolume.balance -apply\nvolume.fix.replication -apply\ns3.clean.uploads -timeAgo=24h",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := cleanMaintenanceScript(tt.input)
if got != tt.expected {
t.Errorf("cleanMaintenanceScript(%q) = %q, want %q", tt.input, got, tt.expected)
}
})
}
}
+5 -8
View File
@@ -324,7 +324,7 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
}
// Only process directories that are versions (start with "v")
if versionResp.Entry != nil && versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
if versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
versionDir := filepath.Join(topicDir, versionResp.Entry.Name)
// List all partition directories under the version directory (e.g., 0315-0630)
@@ -352,7 +352,7 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
}
// Only process directories that are partitions (format: NNNN-NNNN)
if partitionResp.Entry != nil && partitionResp.Entry.IsDirectory {
if partitionResp.Entry.IsDirectory {
// Parse partition range to get partition start ID (e.g., "0315-0630" -> 315)
var partitionStart, partitionStop int32
if n, err := fmt.Sscanf(partitionResp.Entry.Name, "%04d-%04d", &partitionStart, &partitionStop); n != 2 || err != nil {
@@ -387,11 +387,11 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
}
// Only process .offset files
if offsetResp.Entry != nil && !offsetResp.Entry.IsDirectory && strings.HasSuffix(offsetResp.Entry.Name, ".offset") {
if !offsetResp.Entry.IsDirectory && strings.HasSuffix(offsetResp.Entry.Name, ".offset") {
consumerGroup := strings.TrimSuffix(offsetResp.Entry.Name, ".offset")
// Read the offset value from the file
offsetData, err := filer.ReadInsideFiler(context.Background(), client, partitionDir, offsetResp.Entry.Name)
offsetData, err := filer.ReadInsideFiler(client, partitionDir, offsetResp.Entry.Name)
if err != nil {
glog.Warningf("Failed to read offset file %s: %v", offsetResp.Entry.Name, err)
continue
@@ -401,10 +401,7 @@ func (s *AdminServer) GetConsumerGroupOffsets(namespace, topicName string) ([]Co
offset := int64(util.BytesToUint64(offsetData))
// Get the file modification time
var lastUpdated time.Time
if offsetResp.Entry.Attributes != nil {
lastUpdated = time.Unix(offsetResp.Entry.Attributes.Mtime, 0)
}
lastUpdated := time.Unix(offsetResp.Entry.Attributes.Mtime, 0)
offsets = append(offsets, ConsumerGroupOffsetInfo{
ConsumerGroup: consumerGroup,
-208
View File
@@ -5,7 +5,6 @@ import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -131,47 +130,6 @@ func (s *AdminServer) GetPluginJobDetailAPI(w http.ResponseWriter, r *http.Reque
writeJSON(w, http.StatusOK, detail)
}
// ExpirePluginJobAPI marks a job as failed so it no longer blocks scheduling.
func (s *AdminServer) ExpirePluginJobAPI(w http.ResponseWriter, r *http.Request) {
jobID := strings.TrimSpace(mux.Vars(r)["jobId"])
if jobID == "" {
writeJSONError(w, http.StatusBadRequest, "jobId is required")
return
}
var req struct {
Reason string `json:"reason"`
}
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil && err != io.EOF {
writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
return
}
job, expired, err := s.ExpirePluginJob(jobID, req.Reason)
if err != nil {
if errors.Is(err, plugin.ErrJobNotFound) {
writeJSONError(w, http.StatusNotFound, err.Error())
return
}
writeJSONError(w, http.StatusInternalServerError, err.Error())
return
}
response := map[string]interface{}{
"job_id": jobID,
"expired": expired,
}
if job != nil {
response["job"] = job
}
if !expired {
response["message"] = "job is not active"
}
writeJSON(w, http.StatusOK, response)
}
// GetPluginActivitiesAPI returns recent plugin activities.
func (s *AdminServer) GetPluginActivitiesAPI(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
@@ -214,74 +172,6 @@ func (s *AdminServer) GetPluginSchedulerStatesAPI(w http.ResponseWriter, r *http
writeJSON(w, http.StatusOK, states)
}
// GetPluginSchedulerStatusAPI returns scheduler status including in-process jobs and lock state.
func (s *AdminServer) GetPluginSchedulerStatusAPI(w http.ResponseWriter, r *http.Request) {
pluginSvc := s.GetPlugin()
if pluginSvc == nil {
writeJSON(w, http.StatusOK, map[string]interface{}{
"enabled": false,
})
return
}
response := map[string]interface{}{
"enabled": true,
"scheduler": pluginSvc.GetSchedulerStatus(),
}
if s.pluginLock != nil {
response["lock"] = s.pluginLock.Status()
}
writeJSON(w, http.StatusOK, response)
}
// GetPluginSchedulerConfigAPI returns scheduler configuration.
func (s *AdminServer) GetPluginSchedulerConfigAPI(w http.ResponseWriter, r *http.Request) {
pluginSvc := s.GetPlugin()
if pluginSvc == nil {
writeJSONError(w, http.StatusNotFound, "plugin is not enabled")
return
}
writeJSON(w, http.StatusOK, pluginSvc.GetSchedulerConfig())
}
// UpdatePluginSchedulerConfigAPI updates scheduler configuration.
func (s *AdminServer) UpdatePluginSchedulerConfigAPI(w http.ResponseWriter, r *http.Request) {
pluginSvc := s.GetPlugin()
if pluginSvc == nil {
writeJSONError(w, http.StatusNotFound, "plugin is not enabled")
return
}
var req struct {
IdleSleepSeconds *int32 `json:"idle_sleep_seconds"`
}
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil {
if errors.Is(err, io.EOF) {
writeJSONError(w, http.StatusBadRequest, "request body is required")
return
}
writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
return
}
if req.IdleSleepSeconds == nil {
writeJSONError(w, http.StatusBadRequest, "idle_sleep_seconds is required")
return
}
updated, err := pluginSvc.UpdateSchedulerConfig(plugin.SchedulerConfig{
IdleSleepSeconds: *req.IdleSleepSeconds,
})
if err != nil {
writeJSONError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, updated)
}
// RequestPluginJobTypeSchemaAPI asks a worker for one job type schema.
func (s *AdminServer) RequestPluginJobTypeSchemaAPI(w http.ResponseWriter, r *http.Request) {
jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
@@ -345,9 +235,6 @@ func (s *AdminServer) GetPluginJobTypeConfigAPI(w http.ResponseWriter, r *http.R
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
}
}
if descriptor, err := s.LoadPluginJobTypeDescriptor(jobType); err == nil && descriptor != nil {
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
}
renderProtoJSON(w, http.StatusOK, config)
}
@@ -526,14 +413,6 @@ func (s *AdminServer) RunPluginJobTypeAPI(w http.ResponseWriter, r *http.Request
writeJSONError(w, http.StatusBadRequest, "jobType is required")
return
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detect+execute %s", jobType))
if err != nil {
writeJSONError(w, http.StatusInternalServerError, err.Error())
return
}
if releaseLock != nil {
defer releaseLock()
}
var req struct {
ClusterContext json.RawMessage `json:"cluster_context"`
@@ -850,93 +729,6 @@ func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, i
return jobSpec
}
func applyDescriptorDefaultsToPersistedConfig(
config *plugin_pb.PersistedJobTypeConfig,
descriptor *plugin_pb.JobTypeDescriptor,
) {
if config == nil || descriptor == nil {
return
}
if config.AdminConfigValues == nil {
config.AdminConfigValues = map[string]*plugin_pb.ConfigValue{}
}
if config.WorkerConfigValues == nil {
config.WorkerConfigValues = map[string]*plugin_pb.ConfigValue{}
}
if config.AdminRuntime == nil {
config.AdminRuntime = &plugin_pb.AdminRuntimeConfig{}
}
if descriptor.AdminConfigForm != nil {
for key, value := range descriptor.AdminConfigForm.DefaultValues {
if value == nil {
continue
}
current := config.AdminConfigValues[key]
if current == nil {
config.AdminConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
continue
}
if strings.EqualFold(descriptor.JobType, "admin_script") &&
key == "script" &&
isBlankStringConfigValue(current) {
config.AdminConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
}
}
}
if descriptor.WorkerConfigForm != nil {
for key, value := range descriptor.WorkerConfigForm.DefaultValues {
if value == nil {
continue
}
if config.WorkerConfigValues[key] != nil {
continue
}
config.WorkerConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
}
}
if descriptor.AdminRuntimeDefaults != nil {
runtime := config.AdminRuntime
defaults := descriptor.AdminRuntimeDefaults
if runtime.DetectionIntervalSeconds <= 0 {
runtime.DetectionIntervalSeconds = defaults.DetectionIntervalSeconds
}
if runtime.DetectionTimeoutSeconds <= 0 {
runtime.DetectionTimeoutSeconds = defaults.DetectionTimeoutSeconds
}
if runtime.MaxJobsPerDetection <= 0 {
runtime.MaxJobsPerDetection = defaults.MaxJobsPerDetection
}
if runtime.GlobalExecutionConcurrency <= 0 {
runtime.GlobalExecutionConcurrency = defaults.GlobalExecutionConcurrency
}
if runtime.PerWorkerExecutionConcurrency <= 0 {
runtime.PerWorkerExecutionConcurrency = defaults.PerWorkerExecutionConcurrency
}
if runtime.JobTypeMaxRuntimeSeconds <= 0 {
runtime.JobTypeMaxRuntimeSeconds = defaults.JobTypeMaxRuntimeSeconds
}
if runtime.RetryBackoffSeconds <= 0 {
runtime.RetryBackoffSeconds = defaults.RetryBackoffSeconds
}
if runtime.RetryLimit < 0 {
runtime.RetryLimit = defaults.RetryLimit
}
}
}
func isBlankStringConfigValue(value *plugin_pb.ConfigValue) bool {
if value == nil {
return true
}
kind, ok := value.Kind.(*plugin_pb.ConfigValue_StringValue)
if !ok {
return false
}
return strings.TrimSpace(kind.StringValue) == ""
}
func parsePositiveInt(raw string, defaultValue int) int {
value, err := strconv.Atoi(strings.TrimSpace(raw))
if err != nil || value <= 0 {
-189
View File
@@ -1,120 +1,11 @@
package dash
import (
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
func TestExpirePluginJobAPI(t *testing.T) {
makeRequest := func(adminServer *AdminServer, jobID string, body io.Reader) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPost, "/api/plugin/jobs/"+jobID+"/expire", body)
req = mux.SetURLVars(req, map[string]string{"jobId": jobID})
recorder := httptest.NewRecorder()
adminServer.ExpirePluginJobAPI(recorder, req)
return recorder
}
t.Run("empty job id", func(t *testing.T) {
recorder := makeRequest(&AdminServer{}, "", nil)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("expected 400, got %d", recorder.Code)
}
})
t.Run("invalid json", func(t *testing.T) {
recorder := makeRequest(&AdminServer{}, "job-id", strings.NewReader("{"))
if recorder.Code != http.StatusBadRequest {
t.Fatalf("expected 400, got %d", recorder.Code)
}
})
t.Run("job not found", func(t *testing.T) {
adminServer := &AdminServer{
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
return nil, false, plugin.ErrJobNotFound
},
}
recorder := makeRequest(adminServer, "missing", strings.NewReader(`{"reason":"nope"}`))
if recorder.Code != http.StatusNotFound {
t.Fatalf("expected 404, got %d", recorder.Code)
}
var payload map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("failed to unmarshal body: %v", err)
}
if payload["error"] == nil {
t.Fatalf("expected error payload, got %v", payload)
}
})
t.Run("successful expire", func(t *testing.T) {
expected := &plugin.TrackedJob{JobID: "foo", State: "assigned"}
adminServer := &AdminServer{
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
if jobID != "foo" {
return nil, false, errors.New("unexpected")
}
return expected, true, nil
},
}
recorder := makeRequest(adminServer, "foo", strings.NewReader(`{"reason":"cleanup"}`))
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Code)
}
var payload map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["job_id"] != "foo" {
t.Fatalf("expected job_id foo, got %v", payload["job_id"])
}
if expired, ok := payload["expired"].(bool); !ok || !expired {
t.Fatalf("expected expired=true, got %v", payload["expired"])
}
jobData, ok := payload["job"].(map[string]any)
if !ok || jobData["job_id"] != "foo" {
t.Fatalf("expected job info with job_id, got %v", payload["job"])
}
})
t.Run("non-active job", func(t *testing.T) {
adminServer := &AdminServer{
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
return nil, false, nil
},
}
recorder := makeRequest(adminServer, "bar", strings.NewReader(`{"reason":"ignore"}`))
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Code)
}
var payload map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["job_id"] != "bar" {
t.Fatalf("expected job_id bar, got %v", payload["job_id"])
}
if expired, ok := payload["expired"].(bool); !ok || expired {
t.Fatalf("expected expired=false, got %v", payload["expired"])
}
if payload["message"] != "job is not active" {
t.Fatalf("expected message job is not active, got %v", payload["message"])
}
if _, exists := payload["job"]; exists {
t.Fatalf("expected no job payload for non-active job, got %v", payload["job"])
}
})
}
func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
t.Parallel()
@@ -140,83 +31,3 @@ func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
}
}
func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
t.Parallel()
config := &plugin_pb.PersistedJobTypeConfig{
JobType: "admin_script",
AdminConfigValues: map[string]*plugin_pb.ConfigValue{},
WorkerConfigValues: map[string]*plugin_pb.ConfigValue{},
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
}
descriptor := &plugin_pb.JobTypeDescriptor{
JobType: "admin_script",
AdminConfigForm: &plugin_pb.ConfigForm{
DefaultValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.balance -apply"},
},
"run_interval_minutes": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 17},
},
},
},
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
DetectionIntervalSeconds: 60,
DetectionTimeoutSeconds: 300,
},
}
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
script := config.AdminConfigValues["script"]
if script == nil {
t.Fatalf("expected script default to be backfilled")
}
scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
if !ok || scriptKind.StringValue == "" {
t.Fatalf("expected non-empty script default, got=%+v", script)
}
if config.AdminRuntime.DetectionIntervalSeconds != 60 {
t.Fatalf("expected runtime detection interval default to be backfilled")
}
}
func TestApplyDescriptorDefaultsToPersistedConfigReplacesBlankAdminScript(t *testing.T) {
t.Parallel()
config := &plugin_pb.PersistedJobTypeConfig{
JobType: "admin_script",
AdminConfigValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: " "},
},
},
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
}
descriptor := &plugin_pb.JobTypeDescriptor{
JobType: "admin_script",
AdminConfigForm: &plugin_pb.ConfigForm{
DefaultValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.fix.replication -apply"},
},
},
},
}
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
script := config.AdminConfigValues["script"]
if script == nil {
t.Fatalf("expected script config value")
}
scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
if !ok {
t.Fatalf("expected string script config value, got=%T", script.Kind)
}
if scriptKind.StringValue != "volume.fix.replication -apply" {
t.Fatalf("expected blank script to be replaced by default, got=%q", scriptKind.StringValue)
}
}
+2 -9
View File
@@ -151,21 +151,17 @@ func (p *TopicRetentionPurger) purgeTopicData(topicRetention TopicRetentionConfi
}
// Only process directories that are versions (start with "v")
if versionResp.Entry != nil && versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
if versionResp.Entry.IsDirectory && strings.HasPrefix(versionResp.Entry.Name, "v") {
versionTime, err := p.parseVersionTime(versionResp.Entry.Name)
if err != nil {
glog.Warningf("Failed to parse version time from %s: %v", versionResp.Entry.Name, err)
continue
}
var modTime time.Time
if versionResp.Entry.Attributes != nil {
modTime = time.Unix(versionResp.Entry.Attributes.Mtime, 0)
}
versionDirs = append(versionDirs, VersionDirInfo{
Name: versionResp.Entry.Name,
VersionTime: versionTime,
ModTime: modTime,
ModTime: time.Unix(versionResp.Entry.Attributes.Mtime, 0),
})
}
}
@@ -264,9 +260,6 @@ func (p *TopicRetentionPurger) deleteDirectoryRecursively(client filer_pb.Seawee
return fmt.Errorf("failed to receive entries: %w", err)
}
if resp.Entry == nil {
continue
}
entryPath := filepath.Join(dirPath, resp.Entry.Name)
if resp.Entry.IsDirectory {
-4
View File
@@ -229,14 +229,11 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
pluginApi.HandleFunc("/status", h.adminServer.GetPluginStatusAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/workers", h.adminServer.GetPluginWorkersAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/job-types", h.adminServer.GetPluginJobTypesAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/scheduler-config", h.adminServer.GetPluginSchedulerConfigAPI).Methods(http.MethodGet)
pluginApi.Handle("/scheduler-config", wrapWrite(h.adminServer.UpdatePluginSchedulerConfigAPI)).Methods(http.MethodPut)
pluginApi.HandleFunc("/jobs", h.adminServer.GetPluginJobsAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/jobs/{jobId}", h.adminServer.GetPluginJobAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/jobs/{jobId}/detail", h.adminServer.GetPluginJobDetailAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/activities", h.adminServer.GetPluginActivitiesAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/scheduler-states", h.adminServer.GetPluginSchedulerStatesAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/scheduler-status", h.adminServer.GetPluginSchedulerStatusAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/job-types/{jobType}/descriptor", h.adminServer.GetPluginJobTypeDescriptorAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/job-types/{jobType}/schema", h.adminServer.RequestPluginJobTypeSchemaAPI).Methods(http.MethodPost)
pluginApi.HandleFunc("/job-types/{jobType}/config", h.adminServer.GetPluginJobTypeConfigAPI).Methods(http.MethodGet)
@@ -245,7 +242,6 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
pluginApi.Handle("/job-types/{jobType}/detect", wrapWrite(h.adminServer.TriggerPluginDetectionAPI)).Methods(http.MethodPost)
pluginApi.Handle("/job-types/{jobType}/run", wrapWrite(h.adminServer.RunPluginJobTypeAPI)).Methods(http.MethodPost)
pluginApi.Handle("/jobs/execute", wrapWrite(h.adminServer.ExecutePluginJobAPI)).Methods(http.MethodPost)
pluginApi.Handle("/jobs/{jobId}/expire", wrapWrite(h.adminServer.ExpirePluginJobAPI)).Methods(http.MethodPost)
mqApi := api.PathPrefix("/mq").Subrouter()
mqApi.HandleFunc("/topics/{namespace}/{topic}", h.mqHandlers.GetTopicDetailsAPI).Methods(http.MethodGet)
@@ -21,9 +21,6 @@ func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_NoAuth(t *testing.T) {
if !hasRoute(router, http.MethodGet, "/api/plugin/jobs/example/detail") {
t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in no-auth mode")
}
if !hasRoute(router, http.MethodPost, "/api/plugin/jobs/example/expire") {
t.Fatalf("expected POST /api/plugin/jobs/:jobId/expire to be registered in no-auth mode")
}
}
func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) {
@@ -37,9 +34,6 @@ func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) {
if !hasRoute(router, http.MethodGet, "/api/plugin/jobs/example/detail") {
t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in auth mode")
}
if !hasRoute(router, http.MethodPost, "/api/plugin/jobs/example/expire") {
t.Fatalf("expected POST /api/plugin/jobs/:jobId/expire to be registered in auth mode")
}
}
func TestSetupRoutes_RegistersPluginPages_NoAuth(t *testing.T) {
@@ -5,7 +5,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/admin/topology"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
@@ -230,12 +229,6 @@ func (s *MaintenanceIntegration) ScanWithTaskDetectors(volumeMetrics []*types.Vo
continue
}
// Cancel stale pending tasks for this type before re-detection
maintenanceType := s.taskTypeMap[taskType]
if cancelled := s.maintenanceQueue.CancelPendingTasksByType(maintenanceType); cancelled > 0 {
glog.Infof("Cancelled %d stale pending %s tasks before re-detection", cancelled, taskType)
}
glog.V(2).Infof("Running detection for task type: %s", taskType)
results, err := detector.ScanForTasks(filteredMetrics, clusterInfo)
@@ -535,15 +528,10 @@ func (s *MaintenanceIntegration) SyncTask(task *MaintenanceTask) {
// Volume size is not currently used for Balance/Vacuum impact and is not stored in MaintenanceTask
sourceImpact, targetImpact := topology.CalculateTaskStorageImpact(topology.TaskType(string(taskType)), 0)
// Use unified sources and targets from TaskParams.
// Task protos store ServerAddresses (with gRPC port, e.g., "host:port.grpcPort")
// but the topology indexes disks by NodeId (e.g., "host:port").
// Strip the gRPC port suffix via ToHttpAddress() to match the topology key.
// Use unified sources and targets from TaskParams
for _, src := range task.TypedParams.Sources {
resolvedSrc := pb.ServerAddress(src.Node).ToHttpAddress()
glog.V(2).Infof("SyncTask %s: source proto Node=%q resolved to %q, diskId=%d", task.ID, src.Node, resolvedSrc, src.DiskId)
sources = append(sources, topology.TaskSource{
SourceServer: resolvedSrc,
SourceServer: src.Node,
SourceDisk: src.DiskId,
StorageChange: sourceImpact,
})
@@ -551,10 +539,8 @@ func (s *MaintenanceIntegration) SyncTask(task *MaintenanceTask) {
estimatedSize += int64(src.EstimatedSize)
}
for _, target := range task.TypedParams.Targets {
resolvedTarget := pb.ServerAddress(target.Node).ToHttpAddress()
glog.V(2).Infof("SyncTask %s: target proto Node=%q resolved to %q, diskId=%d", task.ID, target.Node, resolvedTarget, target.DiskId)
destinations = append(destinations, topology.TaskDestination{
TargetServer: resolvedTarget,
TargetServer: target.Node,
TargetDisk: target.DiskId,
StorageChange: targetImpact,
})
+2 -49
View File
@@ -4,7 +4,6 @@ import (
"crypto/rand"
"fmt"
"sort"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -231,46 +230,6 @@ func (mq *MaintenanceQueue) hasDuplicateTask(newTask *MaintenanceTask) bool {
return false
}
// CancelPendingTasksByType cancels all pending tasks of a given type.
// This is called before each detection cycle to ensure stale proposals
// from previous cycles are cleaned up before creating new ones.
func (mq *MaintenanceQueue) CancelPendingTasksByType(taskType MaintenanceTaskType) int {
mq.mutex.Lock()
var remaining []*MaintenanceTask
var cancelledSnapshots []*MaintenanceTask
cancelled := 0
for _, task := range mq.pendingTasks {
if task.Type == taskType {
task.Status = TaskStatusCancelled
now := time.Now()
task.CompletedAt = &now
cancelled++
cancelledSnapshots = append(cancelledSnapshots, snapshotTask(task))
glog.V(1).Infof("Cancelled stale pending task %s (%s) for volume %d before re-detection",
task.ID, task.Type, task.VolumeID)
// Release capacity in ActiveTopology and remove pending operation
if mq.integration != nil {
if at := mq.integration.GetActiveTopology(); at != nil {
_ = at.CompleteTask(task.ID)
}
}
mq.removePendingOperation(task.ID)
} else {
remaining = append(remaining, task)
}
}
mq.pendingTasks = remaining
mq.mutex.Unlock()
// Persist cancelled state outside the lock to avoid blocking
for _, snapshot := range cancelledSnapshots {
mq.saveTaskState(snapshot)
}
return cancelled
}
// AddTasksFromResults converts detection results to tasks and adds them to the queue
func (mq *MaintenanceQueue) AddTasksFromResults(results []*TaskDetectionResult) {
for _, result := range results {
@@ -496,8 +455,8 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
task.Status = TaskStatusFailed
task.Error = error
// Check if task should be retried (skip retry for permanent errors)
if task.RetryCount < task.MaxRetries && !isNonRetriableError(error) {
// Check if task should be retried
if task.RetryCount < task.MaxRetries {
// Record unassignment due to failure/retry
if task.WorkerID != "" && len(task.AssignmentHistory) > 0 {
lastAssignment := task.AssignmentHistory[len(task.AssignmentHistory)-1]
@@ -600,12 +559,6 @@ func (mq *MaintenanceQueue) CompleteTask(taskID string, error string) {
}
}
// isNonRetriableError returns true for errors that will never succeed on retry,
// such as when the volume doesn't exist on the source server.
func isNonRetriableError(errMsg string) bool {
return strings.Contains(errMsg, "not found")
}
// UpdateTaskProgress updates the progress of a running task
func (mq *MaintenanceQueue) UpdateTaskProgress(taskID string, progress float64) {
mq.mutex.Lock()
-103
View File
@@ -30,7 +30,6 @@ const (
runsJSONFileName = "runs.json"
trackedJobsJSONFileName = "tracked_jobs.json"
activitiesJSONFileName = "activities.json"
schedulerJSONFileName = "scheduler.json"
defaultDirPerm = 0o755
defaultFilePerm = 0o644
)
@@ -54,7 +53,6 @@ type ConfigStore struct {
memTrackedJobs []TrackedJob
memActivities []JobActivity
memJobDetails map[string]TrackedJob
memScheduler *SchedulerConfig
}
func NewConfigStore(adminDataDir string) (*ConfigStore, error) {
@@ -95,60 +93,6 @@ func (s *ConfigStore) BaseDir() string {
return s.baseDir
}
func (s *ConfigStore) LoadSchedulerConfig() (*SchedulerConfig, error) {
s.mu.RLock()
if !s.configured {
cfg := s.memScheduler
s.mu.RUnlock()
if cfg == nil {
return nil, nil
}
clone := *cfg
return &clone, nil
}
s.mu.RUnlock()
path := filepath.Join(s.baseDir, schedulerJSONFileName)
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, fmt.Errorf("read scheduler config: %w", err)
}
var cfg SchedulerConfig
if err := json.Unmarshal(data, &cfg); err != nil {
return nil, fmt.Errorf("unmarshal scheduler config: %w", err)
}
return &cfg, nil
}
func (s *ConfigStore) SaveSchedulerConfig(config *SchedulerConfig) error {
if config == nil {
return fmt.Errorf("scheduler config is nil")
}
normalized := normalizeSchedulerConfig(*config)
s.mu.Lock()
if !s.configured {
s.memScheduler = &normalized
s.mu.Unlock()
return nil
}
s.mu.Unlock()
payload, err := json.MarshalIndent(normalized, "", " ")
if err != nil {
return fmt.Errorf("marshal scheduler config: %w", err)
}
path := filepath.Join(s.baseDir, schedulerJSONFileName)
if err := os.WriteFile(path, payload, defaultFilePerm); err != nil {
return fmt.Errorf("save scheduler config: %w", err)
}
return nil
}
func (s *ConfigStore) SaveDescriptor(jobType string, descriptor *plugin_pb.JobTypeDescriptor) error {
if descriptor == nil {
return fmt.Errorf("descriptor is nil")
@@ -256,53 +200,6 @@ func (s *ConfigStore) SaveJobTypeConfig(config *plugin_pb.PersistedJobTypeConfig
return nil
}
// SaveJobTypeConfigIfNotExists atomically checks whether a config for the
// given job type already exists and only persists config when none is found.
// Returns true if the config was saved, false if a config already existed.
func (s *ConfigStore) SaveJobTypeConfigIfNotExists(config *plugin_pb.PersistedJobTypeConfig) (bool, error) {
if config == nil {
return false, fmt.Errorf("job type config is nil")
}
if config.JobType == "" {
return false, fmt.Errorf("job type config has empty job_type")
}
sanitizedJobType, err := sanitizeJobType(config.JobType)
if err != nil {
return false, err
}
config.JobType = sanitizedJobType
clone := proto.Clone(config).(*plugin_pb.PersistedJobTypeConfig)
s.mu.Lock()
defer s.mu.Unlock()
if !s.configured {
if _, exists := s.memConfigs[config.JobType]; exists {
return false, nil
}
s.memConfigs[config.JobType] = clone
return true, nil
}
pbPath := filepath.Join(s.baseDir, jobTypesDirName, config.JobType, configPBFileName)
if _, statErr := os.Stat(pbPath); statErr == nil {
return false, nil
}
jobTypeDir, err := s.ensureJobTypeDir(config.JobType)
if err != nil {
return false, err
}
jsonPath := filepath.Join(jobTypeDir, configJSONFileName)
if err := writeProtoFiles(clone, filepath.Join(jobTypeDir, configPBFileName), jsonPath); err != nil {
return false, fmt.Errorf("save job type config for %s: %w", config.JobType, err)
}
return true, nil
}
func (s *ConfigStore) LoadJobTypeConfig(jobType string) (*plugin_pb.PersistedJobTypeConfig, error) {
if _, err := sanitizeJobType(jobType); err != nil {
return nil, err
-75
View File
@@ -208,81 +208,6 @@ func TestConfigStoreMonitorStateRoundTrip(t *testing.T) {
}
}
func TestConfigStoreSaveJobTypeConfigIfNotExists(t *testing.T) {
t.Parallel()
t.Run("in-memory", func(t *testing.T) {
t.Parallel()
store, err := NewConfigStore("")
if err != nil {
t.Fatalf("NewConfigStore: %v", err)
}
testSaveJobTypeConfigIfNotExists(t, store)
})
t.Run("on-disk", func(t *testing.T) {
t.Parallel()
store, err := NewConfigStore(t.TempDir())
if err != nil {
t.Fatalf("NewConfigStore: %v", err)
}
testSaveJobTypeConfigIfNotExists(t, store)
})
}
func testSaveJobTypeConfigIfNotExists(t *testing.T, store *ConfigStore) {
t.Helper()
cfg := &plugin_pb.PersistedJobTypeConfig{
JobType: "admin_script",
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true},
}
// First call should save.
saved, err := store.SaveJobTypeConfigIfNotExists(cfg)
if err != nil {
t.Fatalf("first SaveJobTypeConfigIfNotExists: %v", err)
}
if !saved {
t.Fatal("expected first call to save the config")
}
// Second call with same job type should not save.
saved, err = store.SaveJobTypeConfigIfNotExists(&plugin_pb.PersistedJobTypeConfig{
JobType: "admin_script",
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: false},
})
if err != nil {
t.Fatalf("second SaveJobTypeConfigIfNotExists: %v", err)
}
if saved {
t.Fatal("expected second call to be a no-op")
}
// Verify the original config was preserved.
loaded, err := store.LoadJobTypeConfig("admin_script")
if err != nil {
t.Fatalf("LoadJobTypeConfig: %v", err)
}
if loaded == nil {
t.Fatal("expected config to exist")
}
if !loaded.AdminRuntime.Enabled {
t.Fatal("expected original config (Enabled=true) to be preserved")
}
// Different job type should still save.
saved, err = store.SaveJobTypeConfigIfNotExists(&plugin_pb.PersistedJobTypeConfig{
JobType: "vacuum",
})
if err != nil {
t.Fatalf("SaveJobTypeConfigIfNotExists for different type: %v", err)
}
if !saved {
t.Fatal("expected save for a different job type")
}
}
func TestConfigStoreJobDetailRoundTrip(t *testing.T) {
t.Parallel()
-7
View File
@@ -1,7 +0,0 @@
package plugin
// LockManager provides a shared exclusive lock for admin-managed detection/execution.
// Acquire returns a release function that must be called when the protected work finishes.
type LockManager interface {
Acquire(reason string) (release func(), err error)
}
+14 -162
View File
@@ -24,7 +24,6 @@ const (
defaultHeartbeatInterval = 30
defaultReconnectDelay = 5
defaultPendingSchemaBuffer = 1
adminScriptJobType = "admin_script"
)
type Options struct {
@@ -33,12 +32,6 @@ type Options struct {
SendTimeout time.Duration
SchedulerTick time.Duration
ClusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error)
LockManager LockManager
// ConfigDefaultsProvider is an optional callback invoked when a job type's
// config is being bootstrapped from its descriptor defaults. It can enrich
// or replace the default config before it is persisted. If nil, descriptor
// defaults are used as-is.
ConfigDefaultsProvider func(config *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig
}
// JobTypeInfo contains metadata about a plugin job type.
@@ -59,8 +52,6 @@ type Plugin struct {
schedulerTick time.Duration
clusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error)
configDefaultsProvider func(config *plugin_pb.PersistedJobTypeConfig) *plugin_pb.PersistedJobTypeConfig
lockManager LockManager
schedulerMu sync.Mutex
nextDetectionAt map[string]time.Time
@@ -71,16 +62,6 @@ type Plugin struct {
schedulerExecMu sync.Mutex
schedulerExecReservations map[string]int
adminScriptRunMu sync.RWMutex
schedulerDetectionMu sync.Mutex
schedulerDetection map[string]*schedulerDetectionInfo
schedulerRunMu sync.Mutex
schedulerRun map[string]*schedulerRunInfo
schedulerLoopMu sync.Mutex
schedulerLoopState schedulerLoopState
schedulerConfigMu sync.RWMutex
schedulerConfig SchedulerConfig
schedulerWakeCh chan struct{}
dedupeMu sync.Mutex
recentDedupeByType map[string]map[string]time.Time
@@ -99,8 +80,6 @@ type Plugin struct {
jobsMu sync.RWMutex
jobs map[string]*TrackedJob
// serialize stale job cleanup to avoid duplicate expirations
staleJobsMu sync.Mutex
jobDetailsMu sync.Mutex
@@ -167,8 +146,6 @@ func New(options Options) (*Plugin, error) {
sendTimeout: sendTimeout,
schedulerTick: schedulerTick,
clusterContextProvider: options.ClusterContextProvider,
configDefaultsProvider: options.ConfigDefaultsProvider,
lockManager: options.LockManager,
sessions: make(map[string]*streamSession),
pendingSchema: make(map[string]chan *plugin_pb.ConfigSchemaResponse),
pendingDetection: make(map[string]*pendingDetectionState),
@@ -177,32 +154,14 @@ func New(options Options) (*Plugin, error) {
detectionInFlight: make(map[string]bool),
detectorLeases: make(map[string]string),
schedulerExecReservations: make(map[string]int),
schedulerDetection: make(map[string]*schedulerDetectionInfo),
schedulerRun: make(map[string]*schedulerRunInfo),
recentDedupeByType: make(map[string]map[string]time.Time),
jobs: make(map[string]*TrackedJob),
activities: make([]JobActivity, 0, 256),
persistTicker: time.NewTicker(2 * time.Second),
schedulerWakeCh: make(chan struct{}, 1),
shutdownCh: make(chan struct{}),
}
plugin.ctx, plugin.ctxCancel = context.WithCancel(context.Background())
if cfg, err := plugin.store.LoadSchedulerConfig(); err != nil {
glog.Warningf("Plugin failed to load scheduler config: %v", err)
plugin.schedulerConfig = DefaultSchedulerConfig()
} else if cfg == nil {
defaults := DefaultSchedulerConfig()
plugin.schedulerConfig = defaults
if plugin.store.IsConfigured() {
if err := plugin.store.SaveSchedulerConfig(&defaults); err != nil {
glog.Warningf("Plugin failed to persist scheduler defaults: %v", err)
}
}
} else {
plugin.schedulerConfig = normalizeSchedulerConfig(*cfg)
}
if err := plugin.loadPersistedMonitorState(); err != nil {
glog.Warningf("Plugin failed to load persisted monitoring state: %v", err)
}
@@ -402,14 +361,9 @@ func (r *Plugin) LoadJobTypeConfig(jobType string) (*plugin_pb.PersistedJobTypeC
}
func (r *Plugin) SaveJobTypeConfig(config *plugin_pb.PersistedJobTypeConfig) error {
if err := r.store.SaveJobTypeConfig(config); err != nil {
return err
}
r.wakeScheduler()
return nil
return r.store.SaveJobTypeConfig(config)
}
func (r *Plugin) LoadDescriptor(jobType string) (*plugin_pb.JobTypeDescriptor, error) {
return r.store.LoadDescriptor(jobType)
}
@@ -426,38 +380,6 @@ func (r *Plugin) BaseDir() string {
return r.store.BaseDir()
}
func (r *Plugin) GetSchedulerConfig() SchedulerConfig {
if r == nil {
return DefaultSchedulerConfig()
}
r.schedulerConfigMu.RLock()
cfg := r.schedulerConfig
r.schedulerConfigMu.RUnlock()
return normalizeSchedulerConfig(cfg)
}
func (r *Plugin) UpdateSchedulerConfig(cfg SchedulerConfig) (SchedulerConfig, error) {
if r == nil {
return DefaultSchedulerConfig(), fmt.Errorf("plugin is not initialized")
}
normalized := normalizeSchedulerConfig(cfg)
if err := r.store.SaveSchedulerConfig(&normalized); err != nil {
return SchedulerConfig{}, err
}
r.schedulerConfigMu.Lock()
r.schedulerConfig = normalized
r.schedulerConfigMu.Unlock()
r.wakeScheduler()
return normalized, nil
}
func (r *Plugin) acquireAdminLock(reason string) (func(), error) {
if r == nil || r.lockManager == nil {
return func() {}, nil
}
return r.lockManager.Acquire(reason)
}
// RunDetectionWithReport requests one detector worker and returns proposals with request metadata.
func (r *Plugin) RunDetectionWithReport(
ctx context.Context,
@@ -465,9 +387,6 @@ func (r *Plugin) RunDetectionWithReport(
clusterContext *plugin_pb.ClusterContext,
maxResults int32,
) (*DetectionReport, error) {
releaseGate := r.acquireDetectionExecutionGate(jobType, false)
defer releaseGate()
detector, err := r.pickDetector(jobType)
if err != nil {
return nil, err
@@ -482,10 +401,7 @@ func (r *Plugin) RunDetectionWithReport(
if err != nil {
return nil, err
}
lastCompletedRun := r.loadLastSuccessfulRun(jobType)
if strings.EqualFold(strings.TrimSpace(jobType), adminScriptJobType) {
lastCompletedRun = r.loadLastCompletedRun(jobType)
}
lastSuccessfulRun := r.loadLastSuccessfulRun(jobType)
state := &pendingDetectionState{
complete: make(chan *plugin_pb.DetectionComplete, 1),
@@ -526,7 +442,7 @@ func (r *Plugin) RunDetectionWithReport(
AdminConfigValues: adminConfigValues,
WorkerConfigValues: workerConfigValues,
ClusterContext: clusterContext,
LastSuccessfulRun: lastCompletedRun,
LastSuccessfulRun: lastSuccessfulRun,
MaxResults: maxResults,
},
},
@@ -613,14 +529,11 @@ func (r *Plugin) ExecuteJob(
if job == nil {
return nil, fmt.Errorf("job is nil")
}
jobType := strings.TrimSpace(job.JobType)
if jobType == "" {
if strings.TrimSpace(job.JobType) == "" {
return nil, fmt.Errorf("job_type is required")
}
releaseGate := r.acquireDetectionExecutionGate(jobType, true)
defer releaseGate()
executor, err := r.registry.PickExecutor(jobType)
executor, err := r.registry.PickExecutor(job.JobType)
if err != nil {
return nil, err
}
@@ -628,23 +541,6 @@ func (r *Plugin) ExecuteJob(
return r.executeJobWithExecutor(ctx, executor, job, clusterContext, attempt)
}
func (r *Plugin) acquireDetectionExecutionGate(jobType string, execution bool) func() {
normalizedJobType := strings.ToLower(strings.TrimSpace(jobType))
if execution && normalizedJobType == adminScriptJobType {
r.adminScriptRunMu.Lock()
return func() {
r.adminScriptRunMu.Unlock()
}
}
if normalizedJobType != adminScriptJobType {
r.adminScriptRunMu.RLock()
return func() {
r.adminScriptRunMu.RUnlock()
}
}
return func() {}
}
func (r *Plugin) executeJobWithExecutor(
ctx context.Context,
executor *WorkerSession,
@@ -973,7 +869,6 @@ func (r *Plugin) handleWorkerMessage(workerID string, message *plugin_pb.WorkerT
switch body := message.Body.(type) {
case *plugin_pb.WorkerToAdminMessage_Hello:
r.registry.UpsertFromHello(body.Hello)
r.wakeScheduler()
case *plugin_pb.WorkerToAdminMessage_Heartbeat:
r.registry.UpdateHeartbeat(workerID, body.Heartbeat)
case *plugin_pb.WorkerToAdminMessage_ConfigSchemaResponse:
@@ -1043,6 +938,14 @@ func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *p
return nil
}
existing, err := r.store.LoadJobTypeConfig(jobType)
if err != nil {
return err
}
if existing != nil {
return nil
}
workerDefaults := CloneConfigValueMap(descriptor.WorkerDefaultValues)
if len(workerDefaults) == 0 && descriptor.WorkerConfigForm != nil {
workerDefaults = CloneConfigValueMap(descriptor.WorkerConfigForm.DefaultValues)
@@ -1065,7 +968,6 @@ func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *p
PerWorkerExecutionConcurrency: defaults.PerWorkerExecutionConcurrency,
RetryLimit: defaults.RetryLimit,
RetryBackoffSeconds: defaults.RetryBackoffSeconds,
JobTypeMaxRuntimeSeconds: defaults.JobTypeMaxRuntimeSeconds,
}
}
@@ -1079,22 +981,7 @@ func (r *Plugin) ensureJobTypeConfigFromDescriptor(jobType string, descriptor *p
UpdatedBy: "plugin",
}
// Check existence first to avoid calling configDefaultsProvider unnecessarily
// (e.g., it may make a blocking gRPC call to fetch master config).
existing, err := r.store.LoadJobTypeConfig(jobType)
if err != nil {
return err
}
if existing != nil {
return nil
}
if r.configDefaultsProvider != nil {
cfg = r.configDefaultsProvider(cfg)
}
_, err = r.store.SaveJobTypeConfigIfNotExists(cfg)
return err
return r.store.SaveJobTypeConfig(cfg)
}
func (r *Plugin) handleDetectionProposals(workerID string, message *plugin_pb.DetectionProposals) {
@@ -1402,41 +1289,6 @@ func (r *Plugin) loadLastSuccessfulRun(jobType string) *timestamppb.Timestamp {
return timestamppb.New(latest.UTC())
}
func (r *Plugin) loadLastCompletedRun(jobType string) *timestamppb.Timestamp {
history, err := r.store.LoadRunHistory(jobType)
if err != nil {
glog.Warningf("Plugin failed to load run history for %s: %v", jobType, err)
return nil
}
if history == nil {
return nil
}
var latest time.Time
for i := range history.SuccessfulRuns {
completedAt := history.SuccessfulRuns[i].CompletedAt
if completedAt == nil || completedAt.IsZero() {
continue
}
if latest.IsZero() || completedAt.After(latest) {
latest = *completedAt
}
}
for i := range history.ErrorRuns {
completedAt := history.ErrorRuns[i].CompletedAt
if completedAt == nil || completedAt.IsZero() {
continue
}
if latest.IsZero() || completedAt.After(latest) {
latest = *completedAt
}
}
if latest.IsZero() {
return nil
}
return timestamppb.New(latest.UTC())
}
func CloneConfigValueMap(in map[string]*plugin_pb.ConfigValue) map[string]*plugin_pb.ConfigValue {
if len(in) == 0 {
return map[string]*plugin_pb.ConfigValue{}
-164
View File
@@ -4,10 +4,8 @@ import (
"context"
"errors"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestRunDetectionSendsCancelOnContextDone(t *testing.T) {
@@ -112,165 +110,3 @@ func TestExecuteJobSendsCancelOnContextDone(t *testing.T) {
t.Fatalf("expected context canceled error, got %v", runErr)
}
}
func TestAdminScriptExecutionBlocksOtherDetection(t *testing.T) {
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New plugin error: %v", err)
}
defer pluginSvc.Shutdown()
const adminWorkerID = "worker-admin-script"
const otherWorkerID = "worker-vacuum"
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: adminWorkerID,
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "admin_script", CanExecute: true, MaxExecutionConcurrency: 1},
},
})
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: otherWorkerID,
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "vacuum", CanDetect: true, MaxDetectionConcurrency: 1},
},
})
adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
pluginSvc.putSession(adminSession)
pluginSvc.putSession(otherSession)
adminErrCh := make(chan error, 1)
go func() {
_, runErr := pluginSvc.ExecuteJob(context.Background(), &plugin_pb.JobSpec{
JobId: "job-admin-script-1",
JobType: "admin_script",
}, &plugin_pb.ClusterContext{}, 1)
adminErrCh <- runErr
}()
adminExecMessage := <-adminSession.outgoing
if adminExecMessage.GetExecuteJobRequest() == nil {
t.Fatalf("expected admin_script execute request")
}
detectErrCh := make(chan error, 1)
go func() {
_, runErr := pluginSvc.RunDetection(context.Background(), "vacuum", &plugin_pb.ClusterContext{}, 10)
detectErrCh <- runErr
}()
select {
case unexpected := <-otherSession.outgoing:
t.Fatalf("expected vacuum detection to wait while admin_script runs, got message: %+v", unexpected)
case <-time.After(100 * time.Millisecond):
}
pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
RequestId: adminExecMessage.RequestId,
JobId: "job-admin-script-1",
JobType: "admin_script",
Success: true,
CompletedAt: timestamppb.Now(),
})
if runErr := <-adminErrCh; runErr != nil {
t.Fatalf("admin_script ExecuteJob error: %v", runErr)
}
detectMessage := <-otherSession.outgoing
detectRequest := detectMessage.GetRunDetectionRequest()
if detectRequest == nil {
t.Fatalf("expected vacuum detection request after admin_script completion")
}
pluginSvc.handleDetectionComplete(otherWorkerID, &plugin_pb.DetectionComplete{
RequestId: detectMessage.RequestId,
JobType: "vacuum",
Success: true,
})
if runErr := <-detectErrCh; runErr != nil {
t.Fatalf("vacuum RunDetection error: %v", runErr)
}
}
func TestAdminScriptExecutionBlocksOtherExecution(t *testing.T) {
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New plugin error: %v", err)
}
defer pluginSvc.Shutdown()
const adminWorkerID = "worker-admin-script"
const otherWorkerID = "worker-vacuum"
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: adminWorkerID,
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "admin_script", CanExecute: true, MaxExecutionConcurrency: 1},
},
})
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: otherWorkerID,
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "vacuum", CanExecute: true, MaxExecutionConcurrency: 1},
},
})
adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
pluginSvc.putSession(adminSession)
pluginSvc.putSession(otherSession)
adminErrCh := make(chan error, 1)
go func() {
_, runErr := pluginSvc.ExecuteJob(context.Background(), &plugin_pb.JobSpec{
JobId: "job-admin-script-2",
JobType: "admin_script",
}, &plugin_pb.ClusterContext{}, 1)
adminErrCh <- runErr
}()
adminExecMessage := <-adminSession.outgoing
if adminExecMessage.GetExecuteJobRequest() == nil {
t.Fatalf("expected admin_script execute request")
}
otherErrCh := make(chan error, 1)
go func() {
_, runErr := pluginSvc.ExecuteJob(context.Background(), &plugin_pb.JobSpec{
JobId: "job-vacuum-1",
JobType: "vacuum",
}, &plugin_pb.ClusterContext{}, 1)
otherErrCh <- runErr
}()
select {
case unexpected := <-otherSession.outgoing:
t.Fatalf("expected vacuum execute to wait while admin_script runs, got message: %+v", unexpected)
case <-time.After(100 * time.Millisecond):
}
pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
RequestId: adminExecMessage.RequestId,
JobId: "job-admin-script-2",
JobType: "admin_script",
Success: true,
CompletedAt: timestamppb.Now(),
})
if runErr := <-adminErrCh; runErr != nil {
t.Fatalf("admin_script ExecuteJob error: %v", runErr)
}
otherExecMessage := <-otherSession.outgoing
if otherExecMessage.GetExecuteJobRequest() == nil {
t.Fatalf("expected vacuum execute request after admin_script completion")
}
pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
RequestId: otherExecMessage.RequestId,
JobId: "job-vacuum-1",
JobType: "vacuum",
Success: true,
CompletedAt: timestamppb.Now(),
})
if runErr := <-otherErrCh; runErr != nil {
t.Fatalf("vacuum ExecuteJob error: %v", runErr)
}
}
@@ -195,64 +195,3 @@ func TestRunDetectionWithReportCapturesDetectionActivities(t *testing.T) {
t.Fatalf("expected requested/proposal/completed activities, got stages=%v", stages)
}
}
func TestRunDetectionAdminScriptUsesLastCompletedRun(t *testing.T) {
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New plugin error: %v", err)
}
defer pluginSvc.Shutdown()
jobType := "admin_script"
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: "worker-admin-script",
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
},
})
session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
pluginSvc.putSession(session)
successCompleted := time.Date(2026, 2, 1, 10, 0, 0, 0, time.UTC)
errorCompleted := successCompleted.Add(45 * time.Minute)
if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
Outcome: RunOutcomeSuccess,
CompletedAt: timeToPtr(successCompleted),
}); err != nil {
t.Fatalf("AppendRunRecord success run: %v", err)
}
if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
Outcome: RunOutcomeError,
CompletedAt: timeToPtr(errorCompleted),
}); err != nil {
t.Fatalf("AppendRunRecord error run: %v", err)
}
errCh := make(chan error, 1)
go func() {
_, runErr := pluginSvc.RunDetection(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10)
errCh <- runErr
}()
message := <-session.outgoing
detectRequest := message.GetRunDetectionRequest()
if detectRequest == nil {
t.Fatalf("expected run detection request message")
}
if detectRequest.LastSuccessfulRun == nil {
t.Fatalf("expected last_successful_run to be set")
}
if got := detectRequest.LastSuccessfulRun.AsTime().UTC(); !got.Equal(errorCompleted) {
t.Fatalf("unexpected last_successful_run, got=%s want=%s", got, errorCompleted)
}
pluginSvc.handleDetectionComplete("worker-admin-script", &plugin_pb.DetectionComplete{
RequestId: message.RequestId,
JobType: jobType,
Success: true,
})
if runErr := <-errCh; runErr != nil {
t.Fatalf("RunDetection error: %v", runErr)
}
}
-354
View File
@@ -2,7 +2,6 @@ package plugin
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
@@ -10,18 +9,12 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/timestamppb"
)
const (
maxTrackedJobsTotal = 1000
maxActivityRecords = 4000
maxRelatedJobs = 100
// stale active jobs block dedupe and scheduling; use generous defaults to
// avoid expiring legitimate long-running tasks.
defaultStaleActiveJobTimeout = 24 * time.Hour
defaultOrphanedActiveJobTimeout = 15 * time.Minute
)
var (
@@ -30,14 +23,6 @@ var (
StateCanceled = strings.ToLower(plugin_pb.JobState_JOB_STATE_CANCELED.String())
)
type activeJobSnapshot struct {
jobID string
jobType string
workerID string
requestID string
lastUpdate time.Time
}
// activityLess reports whether activity a occurred after activity b (newest-first order).
// A nil OccurredAt is treated as the zero time.
func activityLess(a, b JobActivity) bool {
@@ -69,13 +54,6 @@ func (r *Plugin) loadPersistedMonitorState() error {
if strings.TrimSpace(job.JobID) == "" {
continue
}
if isActiveTrackedJobState(job.State) {
if detail, detailErr := r.store.LoadJobDetail(job.JobID); detailErr != nil {
glog.Warningf("Plugin failed to load detail snapshot for job %s: %v", job.JobID, detailErr)
} else if detail != nil {
mergeTerminalDetailIntoTracked(&job, detail)
}
}
// Backward compatibility: migrate older inline detail payloads
// out of tracked_jobs.json into dedicated per-job detail files.
if hasTrackedJobRichDetails(job) {
@@ -103,265 +81,6 @@ func (r *Plugin) loadPersistedMonitorState() error {
return nil
}
// ExpireJob marks an active job as failed so it no longer blocks scheduling.
func (r *Plugin) ExpireJob(jobID, reason string) (*TrackedJob, bool, error) {
normalizedJobID := strings.TrimSpace(jobID)
if normalizedJobID == "" {
return nil, false, ErrJobNotFound
}
reason = strings.TrimSpace(reason)
if reason == "" {
reason = "job expired by admin request"
}
var jobType string
var requestID string
active := false
r.jobsMu.RLock()
if tracked := r.jobs[normalizedJobID]; tracked != nil {
jobType = tracked.JobType
requestID = tracked.RequestID
active = isActiveTrackedJobState(tracked.State)
}
r.jobsMu.RUnlock()
if jobType == "" || requestID == "" || !active {
if detail, err := r.store.LoadJobDetail(normalizedJobID); err != nil {
return nil, false, err
} else if detail != nil {
if jobType == "" {
jobType = detail.JobType
}
if requestID == "" {
requestID = detail.RequestID
}
if !active && isActiveTrackedJobState(detail.State) {
active = true
}
}
}
if jobType == "" {
return nil, false, ErrJobNotFound
}
if !active {
current, _ := r.GetTrackedJob(normalizedJobID)
if current == nil {
if detail, err := r.store.LoadJobDetail(normalizedJobID); err == nil && detail != nil {
clone := cloneTrackedJob(*detail)
current = &clone
}
}
return current, false, nil
}
now := time.Now().UTC()
r.handleJobCompleted(&plugin_pb.JobCompleted{
JobId: normalizedJobID,
JobType: jobType,
RequestId: requestID,
Success: false,
ErrorMessage: reason,
CompletedAt: timestamppb.New(now),
})
r.appendActivity(JobActivity{
JobID: normalizedJobID,
JobType: jobType,
RequestID: requestID,
Source: "admin_expire",
Message: reason,
Stage: "expired",
OccurredAt: timeToPtr(now),
})
updated, _ := r.GetTrackedJob(normalizedJobID)
return updated, true, nil
}
// expireStaleJobs marks stale active jobs as failed so they stop blocking new work.
func (r *Plugin) expireStaleJobs(now time.Time) int {
if now.IsZero() {
now = time.Now().UTC()
}
r.staleJobsMu.Lock()
defer r.staleJobsMu.Unlock()
snapshots := r.snapshotActiveJobs()
if len(snapshots) == 0 {
return 0
}
expired := 0
for _, snap := range snapshots {
if snap.lastUpdate.IsZero() {
continue
}
if stale, _, _ := r.evaluateStaleJob(now, snap.workerID, snap.lastUpdate); !stale {
continue
}
reason := r.confirmStaleReason(now, snap.jobID)
if reason == "" {
continue
}
r.handleJobCompleted(&plugin_pb.JobCompleted{
JobId: snap.jobID,
JobType: snap.jobType,
RequestId: snap.requestID,
Success: false,
ErrorMessage: reason,
CompletedAt: timestamppb.New(now),
})
expired++
}
return expired
}
func (r *Plugin) snapshotActiveJobs() []activeJobSnapshot {
r.jobsMu.RLock()
defer r.jobsMu.RUnlock()
if len(r.jobs) == 0 {
return nil
}
out := make([]activeJobSnapshot, 0, len(r.jobs))
for _, job := range r.jobs {
if job == nil {
continue
}
if !isActiveTrackedJobState(job.State) {
continue
}
out = append(out, activeJobSnapshot{
jobID: job.JobID,
jobType: job.JobType,
workerID: job.WorkerID,
requestID: job.RequestID,
lastUpdate: jobLastUpdated(job),
})
}
return out
}
func jobLastUpdated(job *TrackedJob) time.Time {
if job == nil {
return time.Time{}
}
if job.UpdatedAt != nil && !job.UpdatedAt.IsZero() {
return *job.UpdatedAt
}
if job.CreatedAt != nil && !job.CreatedAt.IsZero() {
return *job.CreatedAt
}
return time.Time{}
}
func (r *Plugin) evaluateStaleJob(now time.Time, workerID string, lastUpdate time.Time) (bool, time.Duration, string) {
if lastUpdate.IsZero() {
return false, 0, ""
}
timeout := defaultStaleActiveJobTimeout
reason := fmt.Sprintf("job expired after %s without progress", timeout)
workerID = strings.TrimSpace(workerID)
if workerID == "" {
reason = fmt.Sprintf("job expired after %s without executor assignment", timeout)
} else if !r.isWorkerAvailable(workerID) {
timeout = defaultOrphanedActiveJobTimeout
reason = fmt.Sprintf("job expired after %s without worker heartbeat (worker=%s)", timeout, workerID)
}
if now.Sub(lastUpdate) < timeout {
return false, timeout, reason
}
return true, timeout, reason
}
func (r *Plugin) confirmStaleReason(now time.Time, jobID string) string {
r.jobsMu.RLock()
job := r.jobs[jobID]
if job == nil || !isActiveTrackedJobState(job.State) {
r.jobsMu.RUnlock()
return ""
}
lastUpdate := jobLastUpdated(job)
workerID := job.WorkerID
r.jobsMu.RUnlock()
stale, _, reason := r.evaluateStaleJob(now, workerID, lastUpdate)
if !stale {
return ""
}
return reason
}
func (r *Plugin) isWorkerAvailable(workerID string) bool {
workerID = strings.TrimSpace(workerID)
if workerID == "" {
return false
}
_, ok := r.registry.Get(workerID)
return ok
}
func isTerminalTrackedJobState(state string) bool {
normalized := strings.ToLower(strings.TrimSpace(state))
switch normalized {
case StateSucceeded, StateFailed, StateCanceled:
return true
default:
return false
}
}
func mergeTerminalDetailIntoTracked(tracked *TrackedJob, detail *TrackedJob) {
if tracked == nil || detail == nil {
return
}
if !isTerminalTrackedJobState(detail.State) {
return
}
if !isActiveTrackedJobState(tracked.State) {
return
}
if detail.State != "" {
tracked.State = detail.State
}
if detail.Progress != 0 {
tracked.Progress = detail.Progress
}
if detail.Stage != "" {
tracked.Stage = detail.Stage
}
if detail.Message != "" {
tracked.Message = detail.Message
}
if detail.ErrorMessage != "" {
tracked.ErrorMessage = detail.ErrorMessage
}
if detail.ResultSummary != "" {
tracked.ResultSummary = detail.ResultSummary
}
if detail.CompletedAt != nil && !detail.CompletedAt.IsZero() {
tracked.CompletedAt = detail.CompletedAt
}
if detail.UpdatedAt != nil && !detail.UpdatedAt.IsZero() {
tracked.UpdatedAt = detail.UpdatedAt
}
if tracked.UpdatedAt == nil && tracked.CompletedAt != nil {
tracked.UpdatedAt = tracked.CompletedAt
}
}
func (r *Plugin) ListTrackedJobs(jobType string, state string, limit int) []TrackedJob {
r.jobsMu.RLock()
defer r.jobsMu.RUnlock()
@@ -861,79 +580,6 @@ func (r *Plugin) trackExecutionQueued(job *plugin_pb.JobSpec) {
})
}
func (r *Plugin) cancelQueuedJob(job *plugin_pb.JobSpec, cause error) {
reason := "job canceled"
if cause != nil {
reason = cause.Error()
}
r.markJobCanceled(job, reason)
}
func (r *Plugin) markJobCanceled(job *plugin_pb.JobSpec, reason string) {
if job == nil || strings.TrimSpace(job.JobId) == "" {
return
}
now := time.Now().UTC()
if strings.TrimSpace(reason) == "" {
reason = "job canceled"
}
r.jobsMu.Lock()
tracked := r.jobs[job.JobId]
if tracked == nil {
tracked = &TrackedJob{
JobID: job.JobId,
CreatedAt: timeToPtr(now),
}
r.jobs[job.JobId] = tracked
}
if job.JobType != "" {
tracked.JobType = job.JobType
}
tracked.State = StateCanceled
tracked.Stage = "canceled"
tracked.Message = reason
tracked.ErrorMessage = reason
tracked.Progress = 0
if tracked.CreatedAt == nil || tracked.CreatedAt.IsZero() {
tracked.CreatedAt = timeToPtr(now)
}
tracked.UpdatedAt = timeToPtr(now)
tracked.CompletedAt = timeToPtr(now)
trackedSnapshot := cloneTrackedJob(*tracked)
r.pruneTrackedJobsLocked()
r.dirtyJobs = true
r.jobsMu.Unlock()
r.persistJobDetailSnapshot(job.JobId, func(detail *TrackedJob) {
detail.JobID = job.JobId
if job.JobType != "" {
detail.JobType = job.JobType
}
detail.State = trackedSnapshot.State
detail.Stage = trackedSnapshot.Stage
detail.Message = trackedSnapshot.Message
detail.ErrorMessage = trackedSnapshot.ErrorMessage
detail.Progress = trackedSnapshot.Progress
if detail.CreatedAt == nil || detail.CreatedAt.IsZero() {
detail.CreatedAt = trackedSnapshot.CreatedAt
}
detail.UpdatedAt = trackedSnapshot.UpdatedAt
detail.CompletedAt = trackedSnapshot.CompletedAt
})
r.appendActivity(JobActivity{
JobID: job.JobId,
JobType: job.JobType,
Source: "admin_scheduler",
Message: reason,
Stage: "canceled",
OccurredAt: timeToPtr(now),
})
}
func (r *Plugin) trackExecutionCompletion(completed *plugin_pb.JobCompleted) *TrackedJob {
if completed == nil || strings.TrimSpace(completed.JobId) == "" {
return nil
+132 -469
View File
@@ -13,17 +13,13 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
)
var (
errExecutorAtCapacity = errors.New("executor is at capacity")
errSchedulerShutdown = errors.New("scheduler shutdown")
)
var errExecutorAtCapacity = errors.New("executor is at capacity")
const (
defaultSchedulerTick = 5 * time.Second
defaultScheduledDetectionInterval = 300 * time.Second
defaultScheduledDetectionTimeout = 45 * time.Second
defaultScheduledExecutionTimeout = 90 * time.Second
defaultScheduledJobTypeMaxRuntime = 30 * time.Minute
defaultScheduledMaxResults int32 = 1000
defaultScheduledExecutionConcurrency = 1
defaultScheduledPerWorkerConcurrency = 1
@@ -38,7 +34,6 @@ type schedulerPolicy struct {
DetectionInterval time.Duration
DetectionTimeout time.Duration
ExecutionTimeout time.Duration
JobTypeMaxRuntime time.Duration
RetryBackoff time.Duration
MaxResults int32
ExecutionConcurrency int
@@ -49,71 +44,29 @@ type schedulerPolicy struct {
func (r *Plugin) schedulerLoop() {
defer r.wg.Done()
ticker := time.NewTicker(r.schedulerTick)
defer ticker.Stop()
// Try once immediately on startup.
r.runSchedulerTick()
for {
select {
case <-r.shutdownCh:
return
default:
}
hadJobs := r.runSchedulerIteration()
r.recordSchedulerIterationComplete(hadJobs)
if hadJobs {
continue
}
r.setSchedulerLoopState("", "sleeping")
idleSleep := r.GetSchedulerConfig().IdleSleepDuration()
if nextRun := r.earliestNextDetectionAt(); !nextRun.IsZero() {
if until := time.Until(nextRun); until <= 0 {
idleSleep = 0
} else if until < idleSleep {
idleSleep = until
}
}
if idleSleep <= 0 {
continue
}
timer := time.NewTimer(idleSleep)
select {
case <-r.shutdownCh:
timer.Stop()
return
case <-r.schedulerWakeCh:
if !timer.Stop() {
<-timer.C
}
continue
case <-timer.C:
case <-ticker.C:
r.runSchedulerTick()
}
}
}
func (r *Plugin) runSchedulerIteration() bool {
r.expireStaleJobs(time.Now().UTC())
func (r *Plugin) runSchedulerTick() {
jobTypes := r.registry.DetectableJobTypes()
if len(jobTypes) == 0 {
r.setSchedulerLoopState("", "idle")
return false
}
r.setSchedulerLoopState("", "waiting_for_lock")
releaseLock, err := r.acquireAdminLock("plugin scheduler iteration")
if err != nil {
glog.Warningf("Plugin scheduler failed to acquire lock: %v", err)
r.setSchedulerLoopState("", "idle")
return false
}
if releaseLock != nil {
defer releaseLock()
return
}
active := make(map[string]struct{}, len(jobTypes))
hadJobs := false
for _, jobType := range jobTypes {
active[jobType] = struct{}{}
@@ -126,212 +79,20 @@ func (r *Plugin) runSchedulerIteration() bool {
r.clearSchedulerJobType(jobType)
continue
}
initialDelay := time.Duration(0)
if runInfo := r.snapshotSchedulerRun(jobType); runInfo.lastRunStartedAt.IsZero() {
initialDelay = 5 * time.Second
}
if !r.markDetectionDue(jobType, policy.DetectionInterval, initialDelay) {
if !r.markDetectionDue(jobType, policy.DetectionInterval) {
continue
}
detected := r.runJobTypeIteration(jobType, policy)
if detected {
hadJobs = true
}
r.wg.Add(1)
go func(jt string, p schedulerPolicy) {
defer r.wg.Done()
r.runScheduledDetection(jt, p)
}(jobType, policy)
}
r.pruneSchedulerState(active)
r.pruneDetectorLeases(active)
r.setSchedulerLoopState("", "idle")
return hadJobs
}
func (r *Plugin) wakeScheduler() {
if r == nil {
return
}
select {
case r.schedulerWakeCh <- struct{}{}:
default:
}
}
func (r *Plugin) runJobTypeIteration(jobType string, policy schedulerPolicy) bool {
r.recordSchedulerRunStart(jobType)
r.clearWaitingJobQueue(jobType)
r.setSchedulerLoopState(jobType, "detecting")
r.markJobTypeInFlight(jobType)
defer r.finishDetection(jobType)
start := time.Now().UTC()
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: "scheduled detection started",
Stage: "detecting",
OccurredAt: timeToPtr(start),
})
if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip {
r.recordSchedulerDetectionSkip(jobType, fmt.Sprintf("waiting backlog %d reached threshold %d", waitingCount, waitingThreshold))
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection skipped: waiting backlog %d reached threshold %d", waitingCount, waitingThreshold),
Stage: "skipped_waiting_backlog",
OccurredAt: timeToPtr(time.Now().UTC()),
})
r.recordSchedulerRunComplete(jobType, "skipped")
return false
}
maxRuntime := policy.JobTypeMaxRuntime
if maxRuntime <= 0 {
maxRuntime = defaultScheduledJobTypeMaxRuntime
}
jobCtx, cancel := context.WithTimeout(context.Background(), maxRuntime)
defer cancel()
clusterContext, err := r.loadSchedulerClusterContext(jobCtx)
if err != nil {
r.recordSchedulerDetectionError(jobType, err)
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection aborted: %v", err),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
r.recordSchedulerRunComplete(jobType, "error")
return false
}
detectionTimeout := policy.DetectionTimeout
remaining := time.Until(start.Add(maxRuntime))
if remaining <= 0 {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: "scheduled run timed out before detection",
Stage: "timeout",
OccurredAt: timeToPtr(time.Now().UTC()),
})
r.recordSchedulerRunComplete(jobType, "timeout")
return false
}
if detectionTimeout <= 0 {
detectionTimeout = defaultScheduledDetectionTimeout
}
if detectionTimeout > remaining {
detectionTimeout = remaining
}
detectCtx, cancelDetect := context.WithTimeout(jobCtx, detectionTimeout)
proposals, err := r.RunDetection(detectCtx, jobType, clusterContext, policy.MaxResults)
cancelDetect()
if err != nil {
r.recordSchedulerDetectionError(jobType, err)
stage := "failed"
status := "error"
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
stage = "timeout"
status = "timeout"
}
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection failed: %v", err),
Stage: stage,
OccurredAt: timeToPtr(time.Now().UTC()),
})
r.recordSchedulerRunComplete(jobType, status)
return false
}
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection completed: %d proposal(s)", len(proposals)),
Stage: "detected",
OccurredAt: timeToPtr(time.Now().UTC()),
})
r.recordSchedulerDetectionSuccess(jobType, len(proposals))
detected := len(proposals) > 0
filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
if skippedActive > 0 {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection skipped %d proposal(s) due to active assigned/running jobs", skippedActive),
Stage: "deduped_active_jobs",
OccurredAt: timeToPtr(time.Now().UTC()),
})
}
if len(filteredByActive) == 0 {
r.recordSchedulerRunComplete(jobType, "success")
return detected
}
filtered := r.filterScheduledProposals(filteredByActive)
if len(filtered) != len(filteredByActive) {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection deduped %d proposal(s) within this run", len(filteredByActive)-len(filtered)),
Stage: "deduped",
OccurredAt: timeToPtr(time.Now().UTC()),
})
}
if len(filtered) == 0 {
r.recordSchedulerRunComplete(jobType, "success")
return detected
}
r.setSchedulerLoopState(jobType, "executing")
remaining = time.Until(start.Add(maxRuntime))
if remaining <= 0 {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: "scheduled execution skipped: job type max runtime reached",
Stage: "timeout",
OccurredAt: timeToPtr(time.Now().UTC()),
})
r.recordSchedulerRunComplete(jobType, "timeout")
return detected
}
execPolicy := policy
if execPolicy.ExecutionTimeout <= 0 {
execPolicy.ExecutionTimeout = defaultScheduledExecutionTimeout
}
if execPolicy.ExecutionTimeout > remaining {
execPolicy.ExecutionTimeout = remaining
}
successCount, errorCount, canceledCount := r.dispatchScheduledProposals(jobCtx, jobType, filtered, clusterContext, execPolicy)
status := "success"
if jobCtx.Err() != nil {
status = "timeout"
} else if errorCount > 0 || canceledCount > 0 {
status = "error"
}
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled execution finished: success=%d error=%d canceled=%d", successCount, errorCount, canceledCount),
Stage: "executed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
r.recordSchedulerRunComplete(jobType, status)
return detected
}
func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, error) {
@@ -356,7 +117,6 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
DetectionInterval: durationFromSeconds(adminRuntime.DetectionIntervalSeconds, defaultScheduledDetectionInterval),
DetectionTimeout: durationFromSeconds(adminRuntime.DetectionTimeoutSeconds, defaultScheduledDetectionTimeout),
ExecutionTimeout: defaultScheduledExecutionTimeout,
JobTypeMaxRuntime: durationFromSeconds(adminRuntime.JobTypeMaxRuntimeSeconds, defaultScheduledJobTypeMaxRuntime),
RetryBackoff: durationFromSeconds(adminRuntime.RetryBackoffSeconds, defaultScheduledRetryBackoff),
MaxResults: adminRuntime.MaxJobsPerDetection,
ExecutionConcurrency: int(adminRuntime.GlobalExecutionConcurrency),
@@ -386,9 +146,6 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
if policy.RetryLimit < 0 {
policy.RetryLimit = 0
}
if policy.JobTypeMaxRuntime <= 0 {
policy.JobTypeMaxRuntime = defaultScheduledJobTypeMaxRuntime
}
// Plugin protocol currently has only detection timeout in admin settings.
execTimeout := time.Duration(adminRuntime.DetectionTimeoutSeconds*2) * time.Second
@@ -440,7 +197,6 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
state.DetectionIntervalSeconds = secondsFromDuration(policy.DetectionInterval)
state.DetectionTimeoutSeconds = secondsFromDuration(policy.DetectionTimeout)
state.ExecutionTimeoutSeconds = secondsFromDuration(policy.ExecutionTimeout)
state.JobTypeMaxRuntimeSeconds = secondsFromDuration(policy.JobTypeMaxRuntime)
state.MaxJobsPerDetection = policy.MaxResults
state.GlobalExecutionConcurrency = policy.ExecutionConcurrency
state.PerWorkerExecutionConcurrency = policy.PerWorkerConcurrency
@@ -449,19 +205,6 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
}
}
runInfo := r.snapshotSchedulerRun(jobType)
if !runInfo.lastRunStartedAt.IsZero() {
at := runInfo.lastRunStartedAt
state.LastRunStartedAt = &at
}
if !runInfo.lastRunCompletedAt.IsZero() {
at := runInfo.lastRunCompletedAt
state.LastRunCompletedAt = &at
}
if runInfo.lastRunStatus != "" {
state.LastRunStatus = runInfo.lastRunStatus
}
leasedWorkerID := r.getDetectorLease(jobType)
if leasedWorkerID != "" {
state.DetectorWorkerID = leasedWorkerID
@@ -513,11 +256,10 @@ func deriveSchedulerAdminRuntime(
PerWorkerExecutionConcurrency: defaults.PerWorkerExecutionConcurrency,
RetryLimit: defaults.RetryLimit,
RetryBackoffSeconds: defaults.RetryBackoffSeconds,
JobTypeMaxRuntimeSeconds: defaults.JobTypeMaxRuntimeSeconds,
}
}
func (r *Plugin) markDetectionDue(jobType string, interval, initialDelay time.Duration) bool {
func (r *Plugin) markDetectionDue(jobType string, interval time.Duration) bool {
now := time.Now().UTC()
r.schedulerMu.Lock()
@@ -531,43 +273,12 @@ func (r *Plugin) markDetectionDue(jobType string, interval, initialDelay time.Du
if exists && now.Before(nextRun) {
return false
}
if !exists && initialDelay > 0 {
r.nextDetectionAt[jobType] = now.Add(initialDelay)
return false
}
r.nextDetectionAt[jobType] = now.Add(interval)
r.detectionInFlight[jobType] = true
return true
}
func (r *Plugin) earliestNextDetectionAt() time.Time {
if r == nil {
return time.Time{}
}
r.schedulerMu.Lock()
defer r.schedulerMu.Unlock()
var earliest time.Time
for _, nextRun := range r.nextDetectionAt {
if nextRun.IsZero() {
continue
}
if earliest.IsZero() || nextRun.Before(earliest) {
earliest = nextRun
}
}
return earliest
}
func (r *Plugin) markJobTypeInFlight(jobType string) {
r.schedulerMu.Lock()
r.detectionInFlight[jobType] = true
r.schedulerMu.Unlock()
}
func (r *Plugin) finishDetection(jobType string) {
r.schedulerMu.Lock()
delete(r.detectionInFlight, jobType)
@@ -605,18 +316,105 @@ func (r *Plugin) pruneDetectorLeases(activeJobTypes map[string]struct{}) {
}
}
func (r *Plugin) loadSchedulerClusterContext(ctx context.Context) (*plugin_pb.ClusterContext, error) {
func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) {
defer r.finishDetection(jobType)
start := time.Now().UTC()
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: "scheduled detection started",
Stage: "detecting",
OccurredAt: timeToPtr(start),
})
if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection skipped: waiting backlog %d reached threshold %d", waitingCount, waitingThreshold),
Stage: "skipped_waiting_backlog",
OccurredAt: timeToPtr(time.Now().UTC()),
})
return
}
clusterContext, err := r.loadSchedulerClusterContext()
if err != nil {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection aborted: %v", err),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
return
}
ctx, cancel := context.WithTimeout(context.Background(), policy.DetectionTimeout)
proposals, err := r.RunDetection(ctx, jobType, clusterContext, policy.MaxResults)
cancel()
if err != nil {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection failed: %v", err),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
return
}
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection completed: %d proposal(s)", len(proposals)),
Stage: "detected",
OccurredAt: timeToPtr(time.Now().UTC()),
})
filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
if skippedActive > 0 {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection skipped %d proposal(s) due to active assigned/running jobs", skippedActive),
Stage: "deduped_active_jobs",
OccurredAt: timeToPtr(time.Now().UTC()),
})
}
if len(filteredByActive) == 0 {
return
}
filtered := r.filterScheduledProposals(filteredByActive)
if len(filtered) != len(filteredByActive) {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection deduped %d proposal(s) within this run", len(filteredByActive)-len(filtered)),
Stage: "deduped",
OccurredAt: timeToPtr(time.Now().UTC()),
})
}
if len(filtered) == 0 {
return
}
r.dispatchScheduledProposals(jobType, filtered, clusterContext, policy)
}
func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error) {
if r.clusterContextProvider == nil {
return nil, fmt.Errorf("cluster context provider is not configured")
}
if ctx == nil {
ctx = context.Background()
}
clusterCtx, cancel := context.WithTimeout(ctx, defaultClusterContextTimeout)
ctx, cancel := context.WithTimeout(context.Background(), defaultClusterContextTimeout)
defer cancel()
clusterContext, err := r.clusterContextProvider(clusterCtx)
clusterContext, err := r.clusterContextProvider(ctx)
if err != nil {
return nil, err
}
@@ -627,16 +425,11 @@ func (r *Plugin) loadSchedulerClusterContext(ctx context.Context) (*plugin_pb.Cl
}
func (r *Plugin) dispatchScheduledProposals(
ctx context.Context,
jobType string,
proposals []*plugin_pb.JobProposal,
clusterContext *plugin_pb.ClusterContext,
policy schedulerPolicy,
) (int, int, int) {
if ctx == nil {
ctx = context.Background()
}
) {
jobQueue := make(chan *plugin_pb.JobSpec, len(proposals))
for index, proposal := range proposals {
job := buildScheduledJobSpec(jobType, proposal, index)
@@ -644,7 +437,7 @@ func (r *Plugin) dispatchScheduledProposals(
select {
case <-r.shutdownCh:
close(jobQueue)
return 0, 0, 0
return
default:
jobQueue <- job
}
@@ -655,7 +448,6 @@ func (r *Plugin) dispatchScheduledProposals(
var statsMu sync.Mutex
successCount := 0
errorCount := 0
canceledCount := 0
workerCount := policy.ExecutionConcurrency
if workerCount < 1 {
@@ -667,7 +459,6 @@ func (r *Plugin) dispatchScheduledProposals(
go func() {
defer wg.Done()
jobLoop:
for job := range jobQueue {
select {
case <-r.shutdownCh:
@@ -675,36 +466,19 @@ func (r *Plugin) dispatchScheduledProposals(
default:
}
if ctx.Err() != nil {
r.cancelQueuedJob(job, ctx.Err())
statsMu.Lock()
canceledCount++
statsMu.Unlock()
continue
}
for {
select {
case <-r.shutdownCh:
return
default:
}
if ctx.Err() != nil {
r.cancelQueuedJob(job, ctx.Err())
statsMu.Lock()
canceledCount++
statsMu.Unlock()
continue jobLoop
}
executor, release, reserveErr := r.reserveScheduledExecutor(ctx, jobType, policy)
executor, release, reserveErr := r.reserveScheduledExecutor(jobType, policy)
if reserveErr != nil {
if ctx.Err() != nil {
r.cancelQueuedJob(job, ctx.Err())
statsMu.Lock()
canceledCount++
statsMu.Unlock()
continue jobLoop
select {
case <-r.shutdownCh:
return
default:
}
statsMu.Lock()
errorCount++
@@ -719,23 +493,16 @@ func (r *Plugin) dispatchScheduledProposals(
break
}
err := r.executeScheduledJobWithExecutor(ctx, executor, job, clusterContext, policy)
err := r.executeScheduledJobWithExecutor(executor, job, clusterContext, policy)
release()
if errors.Is(err, errExecutorAtCapacity) {
r.trackExecutionQueued(job)
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
return
}
continue
}
if err != nil {
if ctx.Err() != nil || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
r.cancelQueuedJob(job, err)
statsMu.Lock()
canceledCount++
statsMu.Unlock()
continue jobLoop
}
statsMu.Lock()
errorCount++
statsMu.Unlock()
@@ -761,34 +528,23 @@ func (r *Plugin) dispatchScheduledProposals(
wg.Wait()
drainErr := ctx.Err()
if drainErr == nil {
drainErr = errSchedulerShutdown
}
for job := range jobQueue {
r.cancelQueuedJob(job, drainErr)
canceledCount++
}
return successCount, errorCount, canceledCount
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled execution finished: success=%d error=%d", successCount, errorCount),
Stage: "executed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
}
func (r *Plugin) reserveScheduledExecutor(
ctx context.Context,
jobType string,
policy schedulerPolicy,
) (*WorkerSession, func(), error) {
if ctx == nil {
ctx = context.Background()
}
deadline := time.Now().Add(policy.ExecutionTimeout)
if policy.ExecutionTimeout <= 0 {
deadline = time.Now().Add(10 * time.Minute) // Default cap
}
if ctxDeadline, ok := ctx.Deadline(); ok && ctxDeadline.Before(deadline) {
deadline = ctxDeadline
}
for {
select {
@@ -796,9 +552,6 @@ func (r *Plugin) reserveScheduledExecutor(
return nil, nil, fmt.Errorf("plugin is shutting down")
default:
}
if ctx.Err() != nil {
return nil, nil, ctx.Err()
}
if time.Now().After(deadline) {
return nil, nil, fmt.Errorf("timed out waiting for executor capacity for %s", jobType)
@@ -806,10 +559,7 @@ func (r *Plugin) reserveScheduledExecutor(
executors, err := r.registry.ListExecutors(jobType)
if err != nil {
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
if ctx.Err() != nil {
return nil, nil, ctx.Err()
}
if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
return nil, nil, fmt.Errorf("plugin is shutting down")
}
continue
@@ -823,10 +573,7 @@ func (r *Plugin) reserveScheduledExecutor(
return executor, release, nil
}
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
if ctx.Err() != nil {
return nil, nil, ctx.Err()
}
if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
return nil, nil, fmt.Errorf("plugin is shutting down")
}
}
@@ -911,7 +658,6 @@ func schedulerWorkerExecutionLimit(executor *WorkerSession, jobType string, poli
}
func (r *Plugin) executeScheduledJobWithExecutor(
ctx context.Context,
executor *WorkerSession,
job *plugin_pb.JobSpec,
clusterContext *plugin_pb.ClusterContext,
@@ -929,15 +675,8 @@ func (r *Plugin) executeScheduledJobWithExecutor(
return fmt.Errorf("plugin is shutting down")
default:
}
if ctx != nil && ctx.Err() != nil {
return ctx.Err()
}
parent := ctx
if parent == nil {
parent = context.Background()
}
execCtx, cancel := context.WithTimeout(parent, policy.ExecutionTimeout)
execCtx, cancel := context.WithTimeout(context.Background(), policy.ExecutionTimeout)
_, err := r.executeJobWithExecutor(execCtx, executor, job, clusterContext, int32(attempt))
cancel()
if err == nil {
@@ -957,10 +696,7 @@ func (r *Plugin) executeScheduledJobWithExecutor(
Stage: "retry",
OccurredAt: timeToPtr(time.Now().UTC()),
})
if !waitForShutdownOrTimerWithContext(r.shutdownCh, ctx, policy.RetryBackoff) {
if ctx != nil && ctx.Err() != nil {
return ctx.Err()
}
if !waitForShutdownOrTimer(r.shutdownCh, policy.RetryBackoff) {
return fmt.Errorf("plugin is shutting down")
}
}
@@ -1006,53 +742,6 @@ func (r *Plugin) countWaitingTrackedJobs(jobType string) int {
return waiting
}
func (r *Plugin) clearWaitingJobQueue(jobType string) int {
normalizedJobType := strings.TrimSpace(jobType)
if normalizedJobType == "" {
return 0
}
jobIDs := make([]string, 0)
seen := make(map[string]struct{})
r.jobsMu.RLock()
for _, job := range r.jobs {
if job == nil {
continue
}
if strings.TrimSpace(job.JobType) != normalizedJobType {
continue
}
if !isWaitingTrackedJobState(job.State) {
continue
}
jobID := strings.TrimSpace(job.JobID)
if jobID == "" {
continue
}
if _, ok := seen[jobID]; ok {
continue
}
seen[jobID] = struct{}{}
jobIDs = append(jobIDs, jobID)
}
r.jobsMu.RUnlock()
if len(jobIDs) == 0 {
return 0
}
reason := fmt.Sprintf("cleared queued job before %s run", normalizedJobType)
for _, jobID := range jobIDs {
r.markJobCanceled(&plugin_pb.JobSpec{
JobId: jobID,
JobType: normalizedJobType,
}, reason)
}
return len(jobIDs)
}
func waitingBacklogThreshold(policy schedulerPolicy) int {
concurrency := policy.ExecutionConcurrency
if concurrency <= 0 {
@@ -1150,37 +839,11 @@ func waitForShutdownOrTimer(shutdown <-chan struct{}, duration time.Duration) bo
}
}
func waitForShutdownOrTimerWithContext(shutdown <-chan struct{}, ctx context.Context, duration time.Duration) bool {
if duration <= 0 {
return true
}
if ctx == nil {
ctx = context.Background()
}
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-shutdown:
return false
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
// filterProposalsWithActiveJobs removes proposals whose dedupe keys already have active jobs.
// It first expires stale tracked jobs via expireStaleJobs, which can mutate scheduler state,
// so callers should treat this method as a stateful operation.
func (r *Plugin) filterProposalsWithActiveJobs(jobType string, proposals []*plugin_pb.JobProposal) ([]*plugin_pb.JobProposal, int) {
if len(proposals) == 0 {
return proposals, 0
}
r.expireStaleJobs(time.Now().UTC())
activeKeys := make(map[string]struct{})
r.jobsMu.RLock()
for _, job := range r.jobs {
+5 -18
View File
@@ -1,7 +1,6 @@
package plugin
import (
"context"
"fmt"
"testing"
"time"
@@ -29,7 +28,6 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
PerWorkerExecutionConcurrency: 2,
RetryLimit: 4,
RetryBackoffSeconds: 7,
JobTypeMaxRuntimeSeconds: 1800,
},
})
if err != nil {
@@ -55,9 +53,6 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
if policy.RetryLimit != 4 {
t.Fatalf("unexpected retry limit: got=%d", policy.RetryLimit)
}
if policy.JobTypeMaxRuntime != 30*time.Minute {
t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime)
}
}
func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) {
@@ -80,7 +75,6 @@ func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T
PerWorkerExecutionConcurrency: 2,
RetryLimit: 3,
RetryBackoffSeconds: 6,
JobTypeMaxRuntimeSeconds: 1200,
},
})
if err != nil {
@@ -103,9 +97,6 @@ func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T
if policy.PerWorkerConcurrency != 2 {
t.Fatalf("unexpected per-worker concurrency: got=%d", policy.PerWorkerConcurrency)
}
if policy.JobTypeMaxRuntime != 20*time.Minute {
t.Fatalf("unexpected max runtime: got=%v", policy.JobTypeMaxRuntime)
}
}
func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
@@ -135,13 +126,13 @@ func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
ExecutorReserveBackoff: time.Millisecond,
}
executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
executor1, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy)
if err != nil {
t.Fatalf("reserve executor 1: %v", err)
}
defer release1()
executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
executor2, release2, err := pluginSvc.reserveScheduledExecutor("balance", policy)
if err != nil {
t.Fatalf("reserve executor 2: %v", err)
}
@@ -263,7 +254,7 @@ func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) {
start := time.Now()
pluginSvc.Shutdown()
_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy)
_, _, err = pluginSvc.reserveScheduledExecutor("missing-job-type", policy)
if err == nil {
t.Fatalf("expected reservation shutdown error")
}
@@ -294,7 +285,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
ExecutorReserveBackoff: 5 * time.Millisecond,
}
_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
_, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy)
if err != nil {
t.Fatalf("reserve executor 1: %v", err)
}
@@ -305,7 +296,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
}
secondReserveCh := make(chan reserveResult, 1)
go func() {
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor("balance", policy)
if release2 != nil {
release2()
}
@@ -403,7 +394,6 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
PerWorkerExecutionConcurrency: 2,
RetryLimit: 1,
RetryBackoffSeconds: 9,
JobTypeMaxRuntimeSeconds: 900,
},
})
if err != nil {
@@ -456,9 +446,6 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
if state.ExecutionTimeoutSeconds != 90 {
t.Fatalf("unexpected execution timeout: got=%d", state.ExecutionTimeoutSeconds)
}
if state.JobTypeMaxRuntimeSeconds != 900 {
t.Fatalf("unexpected job type max runtime: got=%d", state.JobTypeMaxRuntimeSeconds)
}
if state.MaxJobsPerDetection != 80 {
t.Fatalf("unexpected max jobs per detection: got=%d", state.MaxJobsPerDetection)
}
-31
View File
@@ -1,31 +0,0 @@
package plugin
import "time"
const (
defaultSchedulerIdleSleep = 613 * time.Second
)
type SchedulerConfig struct {
IdleSleepSeconds int32 `json:"idle_sleep_seconds"`
}
func DefaultSchedulerConfig() SchedulerConfig {
return SchedulerConfig{
IdleSleepSeconds: int32(defaultSchedulerIdleSleep / time.Second),
}
}
func normalizeSchedulerConfig(cfg SchedulerConfig) SchedulerConfig {
if cfg.IdleSleepSeconds <= 0 {
return DefaultSchedulerConfig()
}
return cfg
}
func (c SchedulerConfig) IdleSleepDuration() time.Duration {
if c.IdleSleepSeconds <= 0 {
return defaultSchedulerIdleSleep
}
return time.Duration(c.IdleSleepSeconds) * time.Second
}
-346
View File
@@ -1,346 +0,0 @@
package plugin
import (
"sort"
"strings"
"time"
)
type SchedulerStatus struct {
Now time.Time `json:"now"`
SchedulerTickSeconds int `json:"scheduler_tick_seconds"`
IdleSleepSeconds int `json:"idle_sleep_seconds,omitempty"`
NextDetectionAt *time.Time `json:"next_detection_at,omitempty"`
CurrentJobType string `json:"current_job_type,omitempty"`
CurrentPhase string `json:"current_phase,omitempty"`
LastIterationHadJobs bool `json:"last_iteration_had_jobs,omitempty"`
LastIterationDoneAt *time.Time `json:"last_iteration_done_at,omitempty"`
Waiting []SchedulerWaitingStatus `json:"waiting,omitempty"`
InProcessJobs []SchedulerJobStatus `json:"in_process_jobs,omitempty"`
JobTypes []SchedulerJobTypeStatus `json:"job_types,omitempty"`
}
type SchedulerWaitingStatus struct {
Reason string `json:"reason"`
JobType string `json:"job_type,omitempty"`
Since *time.Time `json:"since,omitempty"`
Until *time.Time `json:"until,omitempty"`
Details map[string]interface{} `json:"details,omitempty"`
}
type SchedulerJobStatus struct {
JobID string `json:"job_id"`
JobType string `json:"job_type"`
State string `json:"state"`
Stage string `json:"stage,omitempty"`
WorkerID string `json:"worker_id,omitempty"`
Message string `json:"message,omitempty"`
Progress float64 `json:"progress,omitempty"`
CreatedAt *time.Time `json:"created_at,omitempty"`
UpdatedAt *time.Time `json:"updated_at,omitempty"`
DurationSeconds float64 `json:"duration_seconds,omitempty"`
}
type SchedulerJobTypeStatus struct {
JobType string `json:"job_type"`
Enabled bool `json:"enabled"`
DetectionInFlight bool `json:"detection_in_flight"`
NextDetectionAt *time.Time `json:"next_detection_at,omitempty"`
DetectionIntervalSeconds int32 `json:"detection_interval_seconds,omitempty"`
LastDetectedAt *time.Time `json:"last_detected_at,omitempty"`
LastDetectedCount int `json:"last_detected_count,omitempty"`
LastDetectionError string `json:"last_detection_error,omitempty"`
LastDetectionSkipped string `json:"last_detection_skipped,omitempty"`
}
type schedulerDetectionInfo struct {
lastDetectedAt time.Time
lastDetectedCount int
lastErrorAt time.Time
lastError string
lastSkippedAt time.Time
lastSkippedReason string
}
type schedulerRunInfo struct {
lastRunStartedAt time.Time
lastRunCompletedAt time.Time
lastRunStatus string
}
type schedulerLoopState struct {
currentJobType string
currentPhase string
lastIterationHadJobs bool
lastIterationCompleted time.Time
}
func (r *Plugin) recordSchedulerDetectionSuccess(jobType string, count int) {
if r == nil {
return
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
info = &schedulerDetectionInfo{}
r.schedulerDetection[jobType] = info
}
info.lastDetectedAt = time.Now().UTC()
info.lastDetectedCount = count
info.lastError = ""
info.lastSkippedReason = ""
}
func (r *Plugin) recordSchedulerDetectionError(jobType string, err error) {
if r == nil {
return
}
if err == nil {
return
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
info = &schedulerDetectionInfo{}
r.schedulerDetection[jobType] = info
}
info.lastErrorAt = time.Now().UTC()
info.lastError = err.Error()
}
func (r *Plugin) recordSchedulerDetectionSkip(jobType string, reason string) {
if r == nil {
return
}
if strings.TrimSpace(reason) == "" {
return
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
info = &schedulerDetectionInfo{}
r.schedulerDetection[jobType] = info
}
info.lastSkippedAt = time.Now().UTC()
info.lastSkippedReason = reason
}
func (r *Plugin) snapshotSchedulerDetection(jobType string) schedulerDetectionInfo {
if r == nil {
return schedulerDetectionInfo{}
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
return schedulerDetectionInfo{}
}
return *info
}
func (r *Plugin) recordSchedulerRunStart(jobType string) {
if r == nil {
return
}
r.schedulerRunMu.Lock()
defer r.schedulerRunMu.Unlock()
info := r.schedulerRun[jobType]
if info == nil {
info = &schedulerRunInfo{}
r.schedulerRun[jobType] = info
}
info.lastRunStartedAt = time.Now().UTC()
info.lastRunStatus = ""
}
func (r *Plugin) recordSchedulerRunComplete(jobType, status string) {
if r == nil {
return
}
r.schedulerRunMu.Lock()
defer r.schedulerRunMu.Unlock()
info := r.schedulerRun[jobType]
if info == nil {
info = &schedulerRunInfo{}
r.schedulerRun[jobType] = info
}
info.lastRunCompletedAt = time.Now().UTC()
info.lastRunStatus = status
}
func (r *Plugin) snapshotSchedulerRun(jobType string) schedulerRunInfo {
if r == nil {
return schedulerRunInfo{}
}
r.schedulerRunMu.Lock()
defer r.schedulerRunMu.Unlock()
info := r.schedulerRun[jobType]
if info == nil {
return schedulerRunInfo{}
}
return *info
}
func (r *Plugin) setSchedulerLoopState(jobType, phase string) {
if r == nil {
return
}
r.schedulerLoopMu.Lock()
r.schedulerLoopState.currentJobType = jobType
r.schedulerLoopState.currentPhase = phase
r.schedulerLoopMu.Unlock()
}
func (r *Plugin) recordSchedulerIterationComplete(hadJobs bool) {
if r == nil {
return
}
r.schedulerLoopMu.Lock()
r.schedulerLoopState.lastIterationHadJobs = hadJobs
r.schedulerLoopState.lastIterationCompleted = time.Now().UTC()
r.schedulerLoopMu.Unlock()
}
func (r *Plugin) snapshotSchedulerLoopState() schedulerLoopState {
if r == nil {
return schedulerLoopState{}
}
r.schedulerLoopMu.Lock()
defer r.schedulerLoopMu.Unlock()
return r.schedulerLoopState
}
func (r *Plugin) GetSchedulerStatus() SchedulerStatus {
now := time.Now().UTC()
loopState := r.snapshotSchedulerLoopState()
schedulerConfig := r.GetSchedulerConfig()
status := SchedulerStatus{
Now: now,
SchedulerTickSeconds: int(secondsFromDuration(r.schedulerTick)),
InProcessJobs: r.listInProcessJobs(now),
IdleSleepSeconds: int(schedulerConfig.IdleSleepSeconds),
CurrentJobType: loopState.currentJobType,
CurrentPhase: loopState.currentPhase,
LastIterationHadJobs: loopState.lastIterationHadJobs,
}
nextDetectionAt := r.earliestNextDetectionAt()
if nextDetectionAt.IsZero() && loopState.currentPhase == "sleeping" && !loopState.lastIterationCompleted.IsZero() {
idleSleep := schedulerConfig.IdleSleepDuration()
if idleSleep > 0 {
nextDetectionAt = loopState.lastIterationCompleted.Add(idleSleep)
}
}
if !nextDetectionAt.IsZero() {
at := nextDetectionAt
status.NextDetectionAt = &at
}
if !loopState.lastIterationCompleted.IsZero() {
at := loopState.lastIterationCompleted
status.LastIterationDoneAt = &at
}
states, err := r.ListSchedulerStates()
if err != nil {
return status
}
waiting := make([]SchedulerWaitingStatus, 0)
jobTypes := make([]SchedulerJobTypeStatus, 0, len(states))
for _, state := range states {
jobType := state.JobType
info := r.snapshotSchedulerDetection(jobType)
jobStatus := SchedulerJobTypeStatus{
JobType: jobType,
Enabled: state.Enabled,
DetectionInFlight: state.DetectionInFlight,
NextDetectionAt: state.NextDetectionAt,
DetectionIntervalSeconds: state.DetectionIntervalSeconds,
}
if !info.lastDetectedAt.IsZero() {
jobStatus.LastDetectedAt = timeToPtr(info.lastDetectedAt)
jobStatus.LastDetectedCount = info.lastDetectedCount
}
if info.lastError != "" {
jobStatus.LastDetectionError = info.lastError
}
if info.lastSkippedReason != "" {
jobStatus.LastDetectionSkipped = info.lastSkippedReason
}
jobTypes = append(jobTypes, jobStatus)
if state.DetectionInFlight {
waiting = append(waiting, SchedulerWaitingStatus{
Reason: "detection_in_flight",
JobType: jobType,
})
} else if state.Enabled && state.NextDetectionAt != nil && now.Before(*state.NextDetectionAt) {
waiting = append(waiting, SchedulerWaitingStatus{
Reason: "next_detection_at",
JobType: jobType,
Until: state.NextDetectionAt,
})
}
}
sort.Slice(jobTypes, func(i, j int) bool {
return jobTypes[i].JobType < jobTypes[j].JobType
})
status.Waiting = waiting
status.JobTypes = jobTypes
return status
}
func (r *Plugin) listInProcessJobs(now time.Time) []SchedulerJobStatus {
active := make([]SchedulerJobStatus, 0)
if r == nil {
return active
}
r.jobsMu.RLock()
for _, job := range r.jobs {
if job == nil {
continue
}
if !isActiveTrackedJobState(job.State) {
continue
}
start := timeToPtr(now)
if job.CreatedAt != nil && !job.CreatedAt.IsZero() {
start = job.CreatedAt
} else if job.UpdatedAt != nil && !job.UpdatedAt.IsZero() {
start = job.UpdatedAt
}
durationSeconds := 0.0
if start != nil {
durationSeconds = now.Sub(*start).Seconds()
}
active = append(active, SchedulerJobStatus{
JobID: job.JobID,
JobType: job.JobType,
State: strings.ToLower(job.State),
Stage: job.Stage,
WorkerID: job.WorkerID,
Message: job.Message,
Progress: job.Progress,
CreatedAt: job.CreatedAt,
UpdatedAt: job.UpdatedAt,
DurationSeconds: durationSeconds,
})
}
r.jobsMu.RUnlock()
sort.Slice(active, func(i, j int) bool {
if active[i].DurationSeconds != active[j].DurationSeconds {
return active[i].DurationSeconds > active[j].DurationSeconds
}
return active[i].JobID < active[j].JobID
})
return active
}
@@ -1,64 +0,0 @@
package plugin
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
func TestGetSchedulerStatusIncludesInProcessJobs(t *testing.T) {
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
pluginSvc.trackExecutionStart("req-1", "worker-a", &plugin_pb.JobSpec{
JobId: "job-1",
JobType: "vacuum",
}, 1)
status := pluginSvc.GetSchedulerStatus()
if len(status.InProcessJobs) != 1 {
t.Fatalf("expected one in-process job, got %d", len(status.InProcessJobs))
}
if status.InProcessJobs[0].JobID != "job-1" {
t.Fatalf("unexpected job id: %s", status.InProcessJobs[0].JobID)
}
}
func TestGetSchedulerStatusIncludesLastDetectionCount(t *testing.T) {
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
const jobType = "vacuum"
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: "worker-a",
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: jobType, CanDetect: true},
},
})
pluginSvc.recordSchedulerDetectionSuccess(jobType, 3)
status := pluginSvc.GetSchedulerStatus()
found := false
for _, jt := range status.JobTypes {
if jt.JobType != jobType {
continue
}
found = true
if jt.LastDetectedCount != 3 {
t.Fatalf("unexpected last detected count: got=%d want=3", jt.LastDetectedCount)
}
if jt.LastDetectedAt == nil {
t.Fatalf("expected last detected at to be set")
}
}
if !found {
t.Fatalf("expected job type status for %s", jobType)
}
}
+1 -10
View File
@@ -1,9 +1,6 @@
package plugin
import (
"errors"
"time"
)
import "time"
const (
// Keep exactly the last 10 successful and last 10 error runs per job type.
@@ -11,8 +8,6 @@ const (
MaxErrorRunHistory = 10
)
var ErrJobNotFound = errors.New("job not found")
type RunOutcome string
const (
@@ -90,7 +85,6 @@ type SchedulerJobTypeState struct {
DetectionIntervalSeconds int32 `json:"detection_interval_seconds,omitempty"`
DetectionTimeoutSeconds int32 `json:"detection_timeout_seconds,omitempty"`
ExecutionTimeoutSeconds int32 `json:"execution_timeout_seconds,omitempty"`
JobTypeMaxRuntimeSeconds int32 `json:"job_type_max_runtime_seconds,omitempty"`
MaxJobsPerDetection int32 `json:"max_jobs_per_detection,omitempty"`
GlobalExecutionConcurrency int `json:"global_execution_concurrency,omitempty"`
PerWorkerExecutionConcurrency int `json:"per_worker_execution_concurrency,omitempty"`
@@ -99,9 +93,6 @@ type SchedulerJobTypeState struct {
DetectorAvailable bool `json:"detector_available"`
DetectorWorkerID string `json:"detector_worker_id,omitempty"`
ExecutorWorkerCount int `json:"executor_worker_count"`
LastRunStartedAt *time.Time `json:"last_run_started_at,omitempty"`
LastRunCompletedAt *time.Time `json:"last_run_completed_at,omitempty"`
LastRunStatus string `json:"last_run_status,omitempty"`
}
func timeToPtr(t time.Time) *time.Time {
+2 -30
View File
@@ -1,9 +1,5 @@
/* SeaweedFS Dashboard Custom Styles */
:root {
--navbar-height: 56px;
}
/* Link colors - muted */
a {
color: #5b7c99;
@@ -16,7 +12,7 @@ a:hover {
/* Sidebar Styles */
.sidebar {
position: fixed;
top: var(--navbar-height);
top: 56px;
bottom: 0;
left: 0;
z-index: 100;
@@ -55,32 +51,13 @@ main {
@media (max-width: 767.98px) {
.sidebar {
top: var(--navbar-height);
padding-top: 0;
width: 240px;
background-color: #f8f9fa !important;
z-index: 1050;
}
.sidebar.show ~ .sidebar-backdrop {
display: block;
top: 5rem;
}
main {
margin-left: 0;
}
}
/* Sidebar backdrop for mobile overlay */
.sidebar-backdrop {
display: none;
position: fixed;
top: var(--navbar-height);
left: 0;
right: 0;
bottom: 0;
background-color: rgba(0, 0, 0, 0.5);
z-index: 1040;
}
/* Custom card styles */
.border-left-primary {
border-left: 0.25rem solid #6b8caf !important;
@@ -285,11 +262,6 @@ main {
box-shadow: 0 0.15rem 1.75rem 0 rgba(58, 59, 69, 0.15) !important;
}
/* Navbar user icon color */
.navbar-toggler .fa-user {
color: rgba(255, 255, 255, 0.75);
}
/* Collapsible menu styles */
.nav-link[data-bs-toggle="collapse"] {
position: relative;
-30
View File
@@ -31,9 +31,6 @@ function initializeDashboard() {
// Set up submenu behavior
setupSubmenuBehavior();
// Set up mobile sidebar behavior
setupMobileSidebar();
}
// HTMX event listeners
@@ -197,33 +194,6 @@ function setupSubmenuBehavior() {
}
// Mobile sidebar toggle and backdrop behavior
function setupMobileSidebar() {
const sidebar = document.getElementById('sidebarMenu');
const backdrop = document.getElementById('sidebarBackdrop');
if (!sidebar || !backdrop) return;
const hideSidebar = () => {
const bsCollapse = bootstrap.Collapse.getInstance(sidebar);
if (bsCollapse) {
bsCollapse.hide();
}
};
// Close sidebar when backdrop is clicked
backdrop.addEventListener('click', hideSidebar);
// Close sidebar when a nav link is clicked (on mobile)
const sidebarToggler = document.querySelector("button[data-bs-target='#sidebarMenu']");
sidebar.querySelectorAll('a.nav-link:not([data-bs-toggle="collapse"])').forEach(function (link) {
link.addEventListener('click', function () {
if (sidebarToggler && getComputedStyle(sidebarToggler).display !== 'none') {
hideSidebar();
}
});
});
}
// Loading indicator functions
function showLoadingIndicator() {
const indicator = document.getElementById('loading-indicator');
-2
View File
@@ -115,7 +115,6 @@ func (at *ActiveTopology) GetDisksWithEffectiveCapacity(taskType TaskType, exclu
RemoteVolumeCount: disk.DiskInfo.DiskInfo.RemoteVolumeCount,
ActiveVolumeCount: disk.DiskInfo.DiskInfo.ActiveVolumeCount,
FreeVolumeCount: disk.DiskInfo.DiskInfo.FreeVolumeCount,
Tags: append([]string(nil), disk.DiskInfo.DiskInfo.Tags...),
}
diskCopy.DiskInfo = diskInfoCopy
diskCopy.DiskInfo.MaxVolumeCount = disk.DiskInfo.DiskInfo.MaxVolumeCount // Ensure Max is set
@@ -179,7 +178,6 @@ func (at *ActiveTopology) GetDisksForPlanning(taskType TaskType, excludeNodeID s
RemoteVolumeCount: disk.DiskInfo.DiskInfo.RemoteVolumeCount,
ActiveVolumeCount: disk.DiskInfo.DiskInfo.ActiveVolumeCount,
FreeVolumeCount: disk.DiskInfo.DiskInfo.FreeVolumeCount,
Tags: append([]string(nil), disk.DiskInfo.DiskInfo.Tags...),
}
diskCopy.DiskInfo = diskInfoCopy
+12 -22
View File
@@ -17,30 +17,20 @@ func (at *ActiveTopology) AssignTask(taskID string) error {
return fmt.Errorf("pending task %s not found", taskID)
}
// Skip capacity check if topology hasn't been populated yet
if len(at.disks) == 0 {
glog.Warningf("AssignTask %s: topology has no disks yet, skipping capacity check", taskID)
} else {
// Check if all destination disks have sufficient capacity to reserve
for _, dest := range task.Destinations {
targetKey := fmt.Sprintf("%s:%d", dest.TargetServer, dest.TargetDisk)
if targetDisk, exists := at.disks[targetKey]; exists {
availableCapacity := at.getEffectiveAvailableCapacityUnsafe(targetDisk)
// Check if all destination disks have sufficient capacity to reserve
for _, dest := range task.Destinations {
targetKey := fmt.Sprintf("%s:%d", dest.TargetServer, dest.TargetDisk)
if targetDisk, exists := at.disks[targetKey]; exists {
availableCapacity := at.getEffectiveAvailableCapacityUnsafe(targetDisk)
// Check if we have enough total capacity using the improved unified comparison
if !availableCapacity.CanAccommodate(dest.StorageChange) {
return fmt.Errorf("insufficient capacity on target disk %s:%d. Available: %+v, Required: %+v",
dest.TargetServer, dest.TargetDisk, availableCapacity, dest.StorageChange)
}
} else if dest.TargetServer != "" {
// Fail fast if destination disk is not found in topology
var existingKeys []string
for k := range at.disks {
existingKeys = append(existingKeys, k)
}
glog.Warningf("destination disk %s not found in topology. Existing disk keys: %v", targetKey, existingKeys)
return fmt.Errorf("destination disk %s not found in topology", targetKey)
// Check if we have enough total capacity using the improved unified comparison
if !availableCapacity.CanAccommodate(dest.StorageChange) {
return fmt.Errorf("insufficient capacity on target disk %s:%d. Available: %+v, Required: %+v",
dest.TargetServer, dest.TargetDisk, availableCapacity, dest.StorageChange)
}
} else if dest.TargetServer != "" {
// Fail fast if destination disk is not found in topology
return fmt.Errorf("destination disk %s not found in topology", targetKey)
}
}
@@ -87,8 +87,6 @@ func (at *ActiveTopology) UpdateTopology(topologyInfo *master_pb.TopologyInfo) e
}
diskKey := fmt.Sprintf("%s:%d", nodeInfo.Id, diskInfo.DiskId)
glog.V(2).Infof("UpdateTopology: adding disk key=%q nodeId=%q diskId=%d diskType=%q address=%q grpcPort=%d volumes=%d maxVolumes=%d",
diskKey, nodeInfo.Id, diskInfo.DiskId, diskType, nodeInfo.Address, nodeInfo.GrpcPort, diskInfo.VolumeCount, diskInfo.MaxVolumeCount)
node.disks[diskInfo.DiskId] = disk
at.disks[diskKey] = disk
}
+14 -158
View File
@@ -65,7 +65,7 @@ templ Plugin(page string) {
<div class="plugin-section plugin-section-overview">
<div class="row mb-4">
<div class="col-md-3 mb-3">
<div class="col-md-4 mb-3">
<div class="card border-info h-100">
<div class="card-body text-center">
<div class="text-uppercase text-muted small mb-1">Workers</div>
@@ -73,7 +73,7 @@ templ Plugin(page string) {
</div>
</div>
</div>
<div class="col-md-3 mb-3">
<div class="col-md-4 mb-3">
<div class="card border-warning h-100">
<div class="card-body text-center">
<div class="text-uppercase text-muted small mb-1">Active Jobs</div>
@@ -81,7 +81,7 @@ templ Plugin(page string) {
</div>
</div>
</div>
<div class="col-md-3 mb-3">
<div class="col-md-4 mb-3">
<div class="card border-success h-100">
<div class="card-body text-center">
<div class="text-uppercase text-muted small mb-1">Activities (recent)</div>
@@ -89,15 +89,6 @@ templ Plugin(page string) {
</div>
</div>
</div>
<div class="col-md-3 mb-3">
<div class="card border-secondary h-100">
<div class="card-body text-center">
<div class="text-uppercase text-muted small mb-1">Next Run</div>
<h4 class="mb-0 plugin-scheduler-next-run">-</h4>
<div class="text-muted small plugin-scheduler-next-run-meta"></div>
</div>
</div>
</div>
</div>
<div class="row mb-4">
<div class="col-12">
@@ -129,7 +120,7 @@ templ Plugin(page string) {
<div class="card shadow-sm">
<div class="card-header d-flex justify-content-between align-items-center flex-wrap gap-2">
<h5 class="mb-0"><i class="fas fa-clock me-2"></i>Scheduler State</h5>
<small class="text-muted">Sequential scheduler with per-job runtime limits</small>
<small class="text-muted">Per job type detection schedule and execution limits</small>
</div>
<div class="card-body p-0">
<div class="table-responsive">
@@ -140,12 +131,12 @@ templ Plugin(page string) {
<th>Enabled</th>
<th>Detector</th>
<th>In Flight</th>
<th>Max Runtime</th>
<th>Next Detection</th>
<th>Interval</th>
<th>Exec Global</th>
<th>Exec/Worker</th>
<th>Executor Workers</th>
<th>Effective Exec</th>
<th>Last Run</th>
</tr>
</thead>
<tbody id="plugin-scheduler-table-body">
@@ -254,16 +245,11 @@ templ Plugin(page string) {
<div class="col-12">
<label class="form-label" for="plugin-admin-detection-interval">Detection Interval (s)</label>
<input type="number" class="form-control" id="plugin-admin-detection-interval" min="0"/>
<div class="form-text">How often to check for new work.</div>
</div>
<div class="col-12">
<label class="form-label" for="plugin-admin-detection-timeout">Detection Timeout (s)</label>
<input type="number" class="form-control" id="plugin-admin-detection-timeout" min="0"/>
</div>
<div class="col-12">
<label class="form-label" for="plugin-admin-max-runtime">Job Type Max Runtime (s)</label>
<input type="number" class="form-control" id="plugin-admin-max-runtime" min="0"/>
</div>
<div class="col-12">
<label class="form-label" for="plugin-admin-max-results">Max Jobs / Detection</label>
<input type="number" class="form-control" id="plugin-admin-max-results" min="0"/>
@@ -287,17 +273,6 @@ templ Plugin(page string) {
</div>
</div>
</div>
<div class="card shadow-sm mt-3">
<div class="card-header d-flex justify-content-between align-items-center">
<h5 class="mb-0"><i class="fas fa-hourglass-half me-2"></i>Next Run</h5>
<small class="text-muted">Scheduler</small>
</div>
<div class="card-body">
<div class="h5 mb-1 plugin-scheduler-next-run">-</div>
<div class="text-muted small plugin-scheduler-next-run-meta">Not scheduled</div>
</div>
</div>
</div>
</div>
@@ -510,11 +485,6 @@ templ Plugin(page string) {
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="plugin-job-detail-modal-label"><i class="fas fa-file-alt me-2"></i>Job Detail</h5>
<div class="ms-auto me-2">
<button type="button" class="btn btn-outline-danger btn-sm" id="plugin-expire-job-btn" disabled>
<i class="fas fa-stop-circle me-1"></i>Expire Job
</button>
</div>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body" id="plugin-job-detail-content">
@@ -536,9 +506,6 @@ templ Plugin(page string) {
.plugin-form-root .card {
border: 1px solid #dee2e6;
}
.plugin-form-root textarea {
min-height: 12rem;
}
.plugin-field-hidden {
display: none;
@@ -597,7 +564,6 @@ templ Plugin(page string) {
jobs: [],
activities: [],
schedulerStates: [],
schedulerStatus: null,
allJobs: [],
allActivities: [],
loadedJobType: '',
@@ -1107,66 +1073,6 @@ templ Plugin(page string) {
return html;
}
function isActiveJobState(candidateState) {
var jobState = candidateState;
if (candidateState && typeof candidateState === 'object' && candidateState.state !== undefined) {
jobState = candidateState.state;
}
var st = String(jobState || '').toLowerCase();
return st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' ||
st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
}
function setExpireButtonState(job) {
var expireBtn = document.getElementById('plugin-expire-job-btn');
if (!expireBtn) {
return;
}
var jobID = job && job.job_id ? String(job.job_id) : '';
var active = isActiveJobState(job);
expireBtn.setAttribute('data-job-id', jobID);
expireBtn.disabled = !jobID || !active;
if (!jobID) {
expireBtn.title = 'Select a job to expire.';
} else if (!active) {
expireBtn.title = 'Job is not active.';
} else {
expireBtn.title = 'Expire job to unblock scheduling.';
}
}
async function expireJob(jobID) {
var normalizedJobID = String(jobID || '').trim();
if (!normalizedJobID) {
return;
}
var reason = window.prompt('Expire job ' + normalizedJobID + '? Optional reason:', 'job expired by admin request');
if (reason === null) {
return;
}
var expireBtn = document.getElementById('plugin-expire-job-btn');
if (expireBtn) {
expireBtn.disabled = true;
}
try {
var response = await pluginRequest('POST', '/api/plugin/jobs/' + encodePath(normalizedJobID) + '/expire', {
reason: reason,
});
if (response && response.expired === false) {
notify(response.message || 'Job is not active.', 'info');
} else {
notify('Job expired: ' + normalizedJobID, 'success');
}
await refreshJobsAndActivities();
await openJobDetail(normalizedJobID);
} catch (e) {
notify('Failed to expire job: ' + e.message, 'error');
}
}
async function openJobDetail(jobID) {
var normalizedJobID = String(jobID || '').trim();
if (!normalizedJobID) {
@@ -1187,12 +1093,10 @@ templ Plugin(page string) {
modal.show();
}
setExpireButtonState(null);
contentRoot.innerHTML = '<div class="text-muted">Loading job detail...</div>';
try {
var detail = await pluginRequest('GET', '/api/plugin/jobs/' + encodePath(normalizedJobID) + '/detail?activity_limit=500&related_limit=20');
var job = (detail && detail.job) ? detail.job : {};
setExpireButtonState(job);
var runRecord = detail && detail.run_record ? detail.run_record : null;
var activities = (detail && Array.isArray(detail.activities)) ? detail.activities : [];
var relatedJobs = (detail && Array.isArray(detail.related_jobs)) ? detail.related_jobs : [];
@@ -1293,7 +1197,6 @@ templ Plugin(page string) {
contentRoot.innerHTML = html;
} catch (e) {
setExpireButtonState(null);
contentRoot.innerHTML = '<div class="alert alert-danger mb-0">Failed to load job detail: ' + escapeHtml(e.message) + '</div>';
}
}
@@ -1335,7 +1238,8 @@ templ Plugin(page string) {
var allActivities = Array.isArray(state.allActivities) ? state.allActivities : [];
var activeCount = allJobs.filter(function(job) {
return isActiveJobState(job);
var st = String(job.state || '').toLowerCase();
return st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' || st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
}).length;
document.getElementById('plugin-status-workers').textContent = String(state.workers.length);
@@ -1361,7 +1265,8 @@ templ Plugin(page string) {
if (!jobType) {
continue;
}
var isActive = isActiveJobState(job);
var st = String(job.state || '').toLowerCase();
var isActive = st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' || st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
if (!isActive) {
continue;
}
@@ -1468,8 +1373,8 @@ templ Plugin(page string) {
var enabled = !!item.enabled;
var inFlight = !!item.detection_in_flight;
var detector = item.detector_available ? textOrDash(item.detector_worker_id) : 'No detector';
var maxRuntimeSeconds = Number(item.job_type_max_runtime_seconds || 0);
var maxRuntimeText = maxRuntimeSeconds > 0 ? (String(maxRuntimeSeconds) + 's') : '-';
var intervalSeconds = Number(item.detection_interval_seconds || 0);
var intervalText = intervalSeconds > 0 ? (String(intervalSeconds) + 's') : '-';
var globalExec = Number(item.global_execution_concurrency || 0);
var perWorkerExec = Number(item.per_worker_execution_concurrency || 0);
var executorWorkers = Number(item.executor_worker_count || 0);
@@ -1478,13 +1383,6 @@ templ Plugin(page string) {
var perWorkerExecText = enabled ? String(perWorkerExec) : '-';
var executorWorkersText = enabled ? String(executorWorkers) : '-';
var effectiveExecText = enabled ? String(effectiveExec) : '-';
var lastRunStatus = textOrDash(item.last_run_status);
var lastRunTime = parseTime(item.last_run_completed_at);
var lastRunText = '-';
if (lastRunStatus !== '-' || lastRunTime) {
var statusLabel = lastRunStatus !== '-' ? lastRunStatus : 'run';
lastRunText = lastRunTime ? (statusLabel + ' @ ' + lastRunTime) : statusLabel;
}
var enabledBadge = enabled ? '<span class="badge bg-success">Enabled</span>' : '<span class="badge bg-secondary">Disabled</span>';
var inFlightBadge = inFlight ? '<span class="badge bg-warning text-dark">Yes</span>' : '<span class="badge bg-light text-dark">No</span>';
@@ -1498,38 +1396,18 @@ templ Plugin(page string) {
'<td>' + enabledBadge + '</td>' +
'<td><small>' + escapeHtml(detector) + '</small></td>' +
'<td>' + inFlightBadge + '</td>' +
'<td><small>' + escapeHtml(maxRuntimeText) + '</small></td>' +
'<td><small>' + escapeHtml(parseTime(item.next_detection_at) || '-') + '</small></td>' +
'<td><small>' + escapeHtml(intervalText) + '</small></td>' +
'<td><small>' + escapeHtml(globalExecText) + '</small></td>' +
'<td><small>' + escapeHtml(perWorkerExecText) + '</small></td>' +
'<td><small>' + escapeHtml(executorWorkersText) + '</small></td>' +
'<td><small>' + escapeHtml(effectiveExecText) + '</small></td>' +
'<td><small>' + escapeHtml(lastRunText) + '</small></td>' +
'</tr>';
}
tbody.innerHTML = rows;
}
function renderSchedulerStatus() {
var valueNodes = document.querySelectorAll('.plugin-scheduler-next-run');
if (!valueNodes.length) {
return;
}
var metaNodes = document.querySelectorAll('.plugin-scheduler-next-run-meta');
var status = state.schedulerStatus || {};
var nextRun = parseTime(status.next_detection_at);
var display = nextRun || '-';
valueNodes.forEach(function(node) {
node.textContent = display;
});
var metaText = nextRun ? 'Local time' : 'Not scheduled';
if (metaNodes.length) {
metaNodes.forEach(function(node) {
node.textContent = metaText;
});
}
}
function renderWorkers() {
var tbody = document.getElementById('plugin-workers-table-body');
if (!state.workers.length) {
@@ -2427,7 +2305,6 @@ templ Plugin(page string) {
document.getElementById('plugin-admin-enabled').checked = pickBool('enabled');
document.getElementById('plugin-admin-detection-interval').value = String(pickNumber('detection_interval_seconds'));
document.getElementById('plugin-admin-detection-timeout').value = String(pickNumber('detection_timeout_seconds'));
document.getElementById('plugin-admin-max-runtime').value = String(pickNumber('job_type_max_runtime_seconds'));
document.getElementById('plugin-admin-max-results').value = String(pickNumber('max_jobs_per_detection'));
document.getElementById('plugin-admin-global-exec').value = String(pickNumber('global_execution_concurrency'));
document.getElementById('plugin-admin-per-worker-exec').value = String(pickNumber('per_worker_execution_concurrency'));
@@ -2452,7 +2329,6 @@ templ Plugin(page string) {
enabled: !!document.getElementById('plugin-admin-enabled').checked,
detection_interval_seconds: getInt('plugin-admin-detection-interval'),
detection_timeout_seconds: getInt('plugin-admin-detection-timeout'),
job_type_max_runtime_seconds: getInt('plugin-admin-max-runtime'),
max_jobs_per_detection: getInt('plugin-admin-max-results'),
global_execution_concurrency: getInt('plugin-admin-global-exec'),
per_worker_execution_concurrency: getInt('plugin-admin-per-worker-exec'),
@@ -2843,30 +2719,21 @@ templ Plugin(page string) {
var allJobsPromise = pluginRequest('GET', '/api/plugin/jobs?limit=500');
var allActivitiesPromise = pluginRequest('GET', '/api/plugin/activities?limit=500');
var schedulerPromise = pluginRequest('GET', '/api/plugin/scheduler-states');
var schedulerStatusPromise = pluginRequest('GET', '/api/plugin/scheduler-status');
var allJobs = await allJobsPromise;
var allActivities = await allActivitiesPromise;
var schedulerStates = await schedulerPromise;
var schedulerStatus = null;
try {
schedulerStatus = await schedulerStatusPromise;
} catch (e) {
schedulerStatus = null;
}
state.jobs = Array.isArray(allJobs) ? allJobs : [];
state.activities = Array.isArray(allActivities) ? allActivities : [];
state.allJobs = state.jobs;
state.allActivities = state.activities;
state.schedulerStates = Array.isArray(schedulerStates) ? schedulerStates : [];
state.schedulerStatus = schedulerStatus && schedulerStatus.scheduler ? schedulerStatus.scheduler : null;
renderQueueJobs();
renderDetectionJobs();
renderExecutionJobs();
renderExecutionActivities();
renderSchedulerStates();
renderSchedulerStatus();
renderStatus();
renderJobTypeSummary();
}
@@ -2911,17 +2778,6 @@ templ Plugin(page string) {
});
}
var expireBtn = document.getElementById('plugin-expire-job-btn');
if (expireBtn) {
expireBtn.addEventListener('click', function() {
var jobID = String(expireBtn.getAttribute('data-job-id') || '').trim();
if (!jobID) {
return;
}
expireJob(jobID);
});
}
document.getElementById('plugin-refresh-all-btn').addEventListener('click', function() {
refreshAll();
});
File diff suppressed because one or more lines are too long
+18 -10
View File
@@ -51,21 +51,19 @@ templ Layout(view ViewContext, content templ.Component) {
<link rel="stylesheet" href="/static/css/admin.css">
</head>
<body>
<div class="container-fluid p-0">
<div class="container-fluid">
<!-- Header -->
<header class="navbar navbar-expand-lg navbar-dark bg-primary sticky-top">
<div class="container-fluid">
<a class="navbar-brand fw-bold" href="/admin">
<i class="fas fa-server me-2"></i>
SeaweedFS Admin
<span class="badge bg-warning text-dark ms-2">ALPHA</span>
</a>
<button class="navbar-toggler d-md-none me-2" type="button" data-bs-toggle="collapse" data-bs-target="#sidebarMenu" aria-controls="sidebarMenu" aria-expanded="false" aria-label="Toggle navigation menu">
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNav">
<span class="navbar-toggler-icon"></span>
</button>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNav" aria-controls="navbarNav" aria-expanded="false" aria-label="User menu">
<i class="fas fa-user"></i>
</button>
<div class="collapse navbar-collapse" id="navbarNav">
<ul class="navbar-nav ms-auto">
@@ -86,7 +84,7 @@ templ Layout(view ViewContext, content templ.Component) {
<div class="row g-0">
<!-- Sidebar -->
<div class="col-md-3 col-lg-2 d-md-block bg-light sidebar collapse" id="sidebarMenu">
<div class="col-md-3 col-lg-2 d-md-block bg-light sidebar collapse">
<div class="position-sticky pt-3">
<h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted">
<span>MAIN</span>
@@ -247,6 +245,19 @@ templ Layout(view ViewContext, content templ.Component) {
</div>
}
</li>
<!-- Commented out for later -->
<!--
<li class="nav-item">
<a class="nav-link" href="/metrics">
<i class="fas fa-chart-line me-2"></i>Metrics
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="/logs">
<i class="fas fa-file-alt me-2"></i>Logs
</a>
</li>
-->
</ul>
<h6 class="sidebar-heading px-3 mt-4 mb-1 text-muted">
@@ -312,11 +323,8 @@ templ Layout(view ViewContext, content templ.Component) {
</div>
</div>
<!-- Sidebar backdrop for mobile -->
<div class="sidebar-backdrop" id="sidebarBackdrop"></div>
<!-- Main content -->
<main class="col-md-9 ms-sm-auto col-lg-10 px-3 px-md-4">
<main class="col-md-9 ms-sm-auto col-lg-10 px-md-4">
<div class="pt-3">
@content
</div>
+13 -13
View File
@@ -71,20 +71,20 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><!-- Bootstrap CSS --><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><!-- Font Awesome CSS --><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"><!-- HTMX --><script src=\"/static/js/htmx.min.js\"></script><!-- Custom CSS --><link rel=\"stylesheet\" href=\"/static/css/admin.css\"></head><body><div class=\"container-fluid p-0\"><!-- Header --><header class=\"navbar navbar-expand-lg navbar-dark bg-primary sticky-top\"><div class=\"container-fluid\"><a class=\"navbar-brand fw-bold\" href=\"/admin\"><i class=\"fas fa-server me-2\"></i> SeaweedFS Admin</a> <button class=\"navbar-toggler d-md-none me-2\" type=\"button\" data-bs-toggle=\"collapse\" data-bs-target=\"#sidebarMenu\" aria-controls=\"sidebarMenu\" aria-expanded=\"false\" aria-label=\"Toggle navigation menu\"><span class=\"navbar-toggler-icon\"></span></button> <button class=\"navbar-toggler\" type=\"button\" data-bs-toggle=\"collapse\" data-bs-target=\"#navbarNav\" aria-controls=\"navbarNav\" aria-expanded=\"false\" aria-label=\"User menu\"><i class=\"fas fa-user\"></i></button><div class=\"collapse navbar-collapse\" id=\"navbarNav\"><ul class=\"navbar-nav ms-auto\"><li class=\"nav-item dropdown\"><a class=\"nav-link dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-user me-1\"></i>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "\"><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><!-- Bootstrap CSS --><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><!-- Font Awesome CSS --><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"><!-- HTMX --><script src=\"/static/js/htmx.min.js\"></script><!-- Custom CSS --><link rel=\"stylesheet\" href=\"/static/css/admin.css\"></head><body><div class=\"container-fluid\"><!-- Header --><header class=\"navbar navbar-expand-lg navbar-dark bg-primary sticky-top\"><div class=\"container-fluid\"><a class=\"navbar-brand fw-bold\" href=\"/admin\"><i class=\"fas fa-server me-2\"></i> SeaweedFS Admin <span class=\"badge bg-warning text-dark ms-2\">ALPHA</span></a> <button class=\"navbar-toggler\" type=\"button\" data-bs-toggle=\"collapse\" data-bs-target=\"#navbarNav\"><span class=\"navbar-toggler-icon\"></span></button><div class=\"collapse navbar-collapse\" id=\"navbarNav\"><ul class=\"navbar-nav ms-auto\"><li class=\"nav-item dropdown\"><a class=\"nav-link dropdown-toggle\" href=\"#\" role=\"button\" data-bs-toggle=\"dropdown\"><i class=\"fas fa-user me-1\"></i>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var3 string
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(username)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 74, Col: 73}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 72, Col: 73}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "</a><ul class=\"dropdown-menu\"><li><a class=\"dropdown-item\" href=\"/logout\"><i class=\"fas fa-sign-out-alt me-2\"></i>Logout</a></li></ul></li></ul></div></div></header><div class=\"row g-0\"><!-- Sidebar --><div class=\"col-md-3 col-lg-2 d-md-block bg-light sidebar collapse\" id=\"sidebarMenu\"><div class=\"position-sticky pt-3\"><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MAIN</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"/admin\"><i class=\"fas fa-tachometer-alt me-2\"></i>Dashboard</a></li><li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "</a><ul class=\"dropdown-menu\"><li><a class=\"dropdown-item\" href=\"/logout\"><i class=\"fas fa-sign-out-alt me-2\"></i>Logout</a></li></ul></li></ul></div></div></header><div class=\"row g-0\"><!-- Sidebar --><div class=\"col-md-3 col-lg-2 d-md-block bg-light sidebar collapse\"><div class=\"position-sticky pt-3\"><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>MAIN</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\"><a class=\"nav-link\" href=\"/admin\"><i class=\"fas fa-tachometer-alt me-2\"></i>Dashboard</a></li><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -113,7 +113,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isClusterPage))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 101, Col: 207}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 99, Col: 207}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
if templ_7745c5c3_Err != nil {
@@ -170,7 +170,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
var templ_7745c5c3_Var11 string
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isStoragePage))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 126, Col: 207}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 124, Col: 207}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
if templ_7745c5c3_Err != nil {
@@ -254,7 +254,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</li></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>WORKERS</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 24, "</li><!-- Commented out for later --><!--\n <li class=\"nav-item\">\n <a class=\"nav-link\" href=\"/metrics\">\n <i class=\"fas fa-chart-line me-2\"></i>Metrics\n </a>\n </li>\n <li class=\"nav-item\">\n <a class=\"nav-link\" href=\"/logs\">\n <i class=\"fas fa-file-alt me-2\"></i>Logs\n </a>\n </li>\n --></ul><h6 class=\"sidebar-heading px-3 mt-4 mb-1 text-muted\"><span>WORKERS</span></h6><ul class=\"nav flex-column\"><li class=\"nav-item\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -329,7 +329,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "</li></ul></div></div><!-- Sidebar backdrop for mobile --><div class=\"sidebar-backdrop\" id=\"sidebarBackdrop\"></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-3 px-md-4\"><div class=\"pt-3\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 39, "</li></ul></div></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-md-4\"><div class=\"pt-3\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -344,7 +344,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
var templ_7745c5c3_Var14 string
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 331, Col: 60}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 339, Col: 60}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
if templ_7745c5c3_Err != nil {
@@ -357,7 +357,7 @@ func Layout(view ViewContext, content templ.Component) templ.Component {
var templ_7745c5c3_Var15 string
templ_7745c5c3_Var15, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 331, Col: 102}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 339, Col: 102}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var15))
if templ_7745c5c3_Err != nil {
@@ -409,7 +409,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
var templ_7745c5c3_Var17 string
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 359, Col: 17}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 367, Col: 17}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
if templ_7745c5c3_Err != nil {
@@ -422,7 +422,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
var templ_7745c5c3_Var18 string
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 373, Col: 57}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 381, Col: 57}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
if templ_7745c5c3_Err != nil {
@@ -440,7 +440,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
var templ_7745c5c3_Var19 string
templ_7745c5c3_Var19, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 380, Col: 45}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 388, Col: 45}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var19))
if templ_7745c5c3_Err != nil {
@@ -458,7 +458,7 @@ func LoginForm(title string, errorMessage string, csrfToken string) templ.Compon
var templ_7745c5c3_Var20 string
templ_7745c5c3_Var20, templ_7745c5c3_Err = templ.JoinStringErrs(csrfToken)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 385, Col: 84}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `view/layout/layout.templ`, Line: 393, Col: 84}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var20))
if templ_7745c5c3_Err != nil {
-6
View File
@@ -1,6 +0,0 @@
package cluster
const (
AdminShellLockName = "shell"
AdminServerPresenceLockName = "admin-server"
)
@@ -1,7 +1,5 @@
package maintenance
const DefaultMaintenanceSleepMinutes = 17
const DefaultMasterMaintenanceScripts = `
lock
ec.encode -fullPercent=95 -quietFor=1h
+1 -2
View File
@@ -45,7 +45,7 @@ const (
defaultMiniVolumeSizeMB = 128 // Default volume size for mini mode
maxVolumeSizeMB = 1024 // Maximum volume size in MB (1GB)
GrpcPortOffset = 10000 // Offset used to calculate gRPC port from HTTP port
defaultMiniPluginJobTypes = "vacuum,volume_balance,erasure_coding,admin_script"
defaultMiniPluginJobTypes = "vacuum,volume_balance,erasure_coding"
)
var (
@@ -204,7 +204,6 @@ func initMiniVolumeFlags() {
miniOptions.v.publicUrl = cmdMini.Flag.String("volume.publicUrl", "", "publicly accessible address")
miniOptions.v.indexType = cmdMini.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
miniOptions.v.diskType = cmdMini.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
miniOptions.v.tags = cmdMini.Flag.String("volume.tags", "", "comma-separated tag groups per data dir; each group uses ':' (e.g. fast:ssd,archive)")
miniOptions.v.fixJpgOrientation = cmdMini.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
miniOptions.v.readMode = cmdMini.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
miniOptions.v.compactionMBPerSecond = cmdMini.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
+5 -13
View File
@@ -1,21 +1,13 @@
package command
import (
"testing"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
import "testing"
func TestMiniDefaultPluginJobTypes(t *testing.T) {
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
// defaultMiniPluginJobTypes is an explicit list: "vacuum,volume_balance,erasure_coding,admin_script"
handlers, err := buildPluginWorkerHandlers(defaultMiniPluginJobTypes, dialOption, int(pluginworker.DefaultMaxExecutionConcurrency), "")
jobTypes, err := parsePluginWorkerJobTypes(defaultMiniPluginJobTypes)
if err != nil {
t.Fatalf("buildPluginWorkerHandlers(mini default) err = %v", err)
t.Fatalf("parsePluginWorkerJobTypes(mini default) err = %v", err)
}
if len(handlers) != 4 {
t.Fatalf("expected mini default job types to include 4 handlers, got %d", len(handlers))
if len(jobTypes) != 3 {
t.Fatalf("expected mini default job types to include 3 handlers, got %v", jobTypes)
}
}
+67 -103
View File
@@ -14,46 +14,68 @@ import (
"google.golang.org/grpc/credentials/insecure"
)
func TestBuildPluginWorkerHandlerExplicitTypes(t *testing.T) {
func TestBuildPluginWorkerHandler(t *testing.T) {
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
testMaxConcurrency := int(pluginworker.DefaultMaxExecutionConcurrency)
for _, jobType := range []string{"vacuum", "volume_balance", "erasure_coding", "admin_script", "iceberg_maintenance"} {
handlers, err := buildPluginWorkerHandlers(jobType, dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandlers(%s) err = %v", jobType, err)
}
if len(handlers) != 1 {
t.Fatalf("expected 1 handler for %s, got %d", jobType, len(handlers))
}
handler, err := buildPluginWorkerHandler("vacuum", dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandler(vacuum) err = %v", err)
}
}
func TestBuildPluginWorkerHandlerAliases(t *testing.T) {
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
testMaxConcurrency := int(pluginworker.DefaultMaxExecutionConcurrency)
for _, alias := range []string{"balance", "ec", "iceberg", "admin", "script"} {
handlers, err := buildPluginWorkerHandlers(alias, dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandlers(%s) err = %v", alias, err)
}
if len(handlers) != 1 {
t.Fatalf("expected 1 handler for alias %s, got %d", alias, len(handlers))
}
if handler == nil {
t.Fatalf("expected non-nil handler")
}
}
func TestBuildPluginWorkerHandlerUnknown(t *testing.T) {
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
_, err := buildPluginWorkerHandlers("unknown", dialOption, 1, "")
handler, err = buildPluginWorkerHandler("", dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandler(default) err = %v", err)
}
if handler == nil {
t.Fatalf("expected non-nil default handler")
}
handler, err = buildPluginWorkerHandler("volume_balance", dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandler(volume_balance) err = %v", err)
}
if handler == nil {
t.Fatalf("expected non-nil volume_balance handler")
}
handler, err = buildPluginWorkerHandler("balance", dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandler(balance alias) err = %v", err)
}
if handler == nil {
t.Fatalf("expected non-nil balance alias handler")
}
handler, err = buildPluginWorkerHandler("erasure_coding", dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandler(erasure_coding) err = %v", err)
}
if handler == nil {
t.Fatalf("expected non-nil erasure_coding handler")
}
handler, err = buildPluginWorkerHandler("ec", dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandler(ec alias) err = %v", err)
}
if handler == nil {
t.Fatalf("expected non-nil ec alias handler")
}
_, err = buildPluginWorkerHandler("unknown", dialOption, testMaxConcurrency, "")
if err == nil {
t.Fatalf("expected error for unknown job type")
t.Fatalf("expected unsupported job type error")
}
}
func TestBuildPluginWorkerHandlers(t *testing.T) {
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
testMaxConcurrency := int(pluginworker.DefaultMaxExecutionConcurrency)
handlers, err := buildPluginWorkerHandlers("vacuum,volume_balance,erasure_coding", dialOption, testMaxConcurrency, "")
@@ -78,97 +100,39 @@ func TestBuildPluginWorkerHandlers(t *testing.T) {
}
}
func TestBuildPluginWorkerHandlersCategories(t *testing.T) {
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
testMaxConcurrency := int(pluginworker.DefaultMaxExecutionConcurrency)
allHandlers, err := buildPluginWorkerHandlers("all", dialOption, testMaxConcurrency, "")
func TestParsePluginWorkerJobTypes(t *testing.T) {
jobTypes, err := parsePluginWorkerJobTypes("")
if err != nil {
t.Fatalf("buildPluginWorkerHandlers(all) err = %v", err)
t.Fatalf("parsePluginWorkerJobTypes(default) err = %v", err)
}
// "all" must include at least vacuum and erasure_coding (one default, one heavy)
allNames := handlerJobTypes(allHandlers)
for _, required := range []string{"vacuum", "erasure_coding", "iceberg_maintenance"} {
if !allNames[required] {
t.Fatalf("'all' missing expected job type %q, got %v", required, allNames)
}
if len(jobTypes) != 1 || jobTypes[0] != "vacuum" {
t.Fatalf("expected default [vacuum], got %v", jobTypes)
}
defaultHandlers, err := buildPluginWorkerHandlers("default", dialOption, testMaxConcurrency, "")
jobTypes, err = parsePluginWorkerJobTypes(" volume_balance , ec , vacuum , volume_balance ")
if err != nil {
t.Fatalf("buildPluginWorkerHandlers(default) err = %v", err)
t.Fatalf("parsePluginWorkerJobTypes(list) err = %v", err)
}
defaultNames := handlerJobTypes(defaultHandlers)
heavyHandlers, err := buildPluginWorkerHandlers("heavy", dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandlers(heavy) err = %v", err)
if len(jobTypes) != 3 {
t.Fatalf("expected 3 deduped job types, got %d (%v)", len(jobTypes), jobTypes)
}
heavyNames := handlerJobTypes(heavyHandlers)
// default and heavy must both be non-empty subsets of all
if len(defaultNames) == 0 {
t.Fatalf("'default' resolved no handlers")
}
if len(heavyNames) == 0 {
t.Fatalf("'heavy' resolved no handlers")
}
for name := range defaultNames {
if !allNames[name] {
t.Fatalf("default handler %q not in 'all'", name)
}
}
for name := range heavyNames {
if !allNames[name] {
t.Fatalf("heavy handler %q not in 'all'", name)
}
if jobTypes[0] != "volume_balance" || jobTypes[1] != "erasure_coding" || jobTypes[2] != "vacuum" {
t.Fatalf("unexpected parsed order %v", jobTypes)
}
// default and heavy must be disjoint and their union must equal all
for name := range defaultNames {
if heavyNames[name] {
t.Fatalf("handler %q appears in both default and heavy", name)
}
}
if len(defaultNames)+len(heavyNames) != len(allNames) {
t.Fatalf("union(default=%d, heavy=%d) != all(%d)", len(defaultNames), len(heavyNames), len(allNames))
}
// mix category + explicit: "default,iceberg" adds one heavy to default set
mixedHandlers, err := buildPluginWorkerHandlers("default,iceberg", dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandlers(default,iceberg) err = %v", err)
}
if len(mixedHandlers) != len(defaultHandlers)+1 {
t.Fatalf("expected default+1 handlers for 'default,iceberg', got %d (default=%d)", len(mixedHandlers), len(defaultHandlers))
if _, err = parsePluginWorkerJobTypes(" , "); err != nil {
t.Fatalf("expected empty list to resolve to default vacuum: %v", err)
}
}
func TestPluginWorkerDefaultJobTypes(t *testing.T) {
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
testMaxConcurrency := int(pluginworker.DefaultMaxExecutionConcurrency)
// defaultPluginWorkerJobTypes is "all", so it should match the "all" category exactly
defaultHandlers, err := buildPluginWorkerHandlers(defaultPluginWorkerJobTypes, dialOption, testMaxConcurrency, "")
jobTypes, err := parsePluginWorkerJobTypes(defaultPluginWorkerJobTypes)
if err != nil {
t.Fatalf("buildPluginWorkerHandlers(default setting) err = %v", err)
t.Fatalf("parsePluginWorkerJobTypes(default setting) err = %v", err)
}
allHandlers, err := buildPluginWorkerHandlers("all", dialOption, testMaxConcurrency, "")
if err != nil {
t.Fatalf("buildPluginWorkerHandlers(all) err = %v", err)
if len(jobTypes) != 3 {
t.Fatalf("expected default job types to include 3 handlers, got %v", jobTypes)
}
if len(defaultHandlers) != len(allHandlers) {
t.Fatalf("default setting resolved %d handlers, 'all' resolved %d", len(defaultHandlers), len(allHandlers))
}
}
// handlerJobTypes returns the set of job type names from a slice of handlers.
func handlerJobTypes(handlers []pluginworker.JobHandler) map[string]bool {
m := make(map[string]bool, len(handlers))
for _, h := range handlers {
m[h.Capability().JobType] = true
}
return m
}
func TestResolvePluginWorkerID(t *testing.T) {
-1
View File
@@ -6,7 +6,6 @@
[master.maintenance]
# periodically run these scripts are the same as running them from 'weed shell'
# Scripts are skipped while an admin server is connected.
scripts = """{{DEFAULT_MAINTENANCE_SCRIPTS}}"""
sleep_minutes = 17 # sleep minutes between each script execution
-1
View File
@@ -137,7 +137,6 @@ func init() {
serverOptions.v.id = cmdServer.Flag.String("volume.id", "", "volume server id. If empty, default to ip:port")
serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
serverOptions.v.diskType = cmdServer.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
serverOptions.v.tags = cmdServer.Flag.String("volume.tags", "", "comma-separated tag groups per data dir; each group uses ':' (e.g. fast:ssd,archive)")
serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
serverOptions.v.readMode = cmdServer.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
+1 -39
View File
@@ -53,7 +53,6 @@ type VolumeServerOptions struct {
whiteList []string
indexType *string
diskType *string
tags *string
fixJpgOrientation *bool
readMode *string
cpuProfile *string
@@ -95,7 +94,6 @@ func init() {
v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
v.diskType = cmdVolume.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
v.tags = cmdVolume.Flag.String("tags", "", "comma-separated tag groups per data dir; each group uses ':' (e.g. fast:ssd,archive)")
v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
v.readMode = cmdVolume.Flag.String("readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|proxy to remote node|redirect volume location'.")
v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
@@ -221,12 +219,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
glog.Fatalf("%d directories by -dir, but only %d disk types is set by -disk", len(v.folders), len(diskTypes))
}
var tagsArg string
if v.tags != nil {
tagsArg = *v.tags
}
folderTags := parseVolumeTags(tagsArg, len(v.folders))
// security related white list configuration
v.whiteList = util.StringSplit(volumeWhiteListOption, ",")
@@ -277,7 +269,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
*v.ip, *v.port, *v.portGrpc, *v.publicUrl, volumeServerId,
v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes, folderTags,
v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
*v.idxFolder,
volumeNeedleMapKind,
v.masters, constants.VolumePulsePeriod, *v.dataCenter, *v.rack,
@@ -342,36 +334,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
}
func parseVolumeTags(tagsArg string, folderCount int) [][]string {
if folderCount <= 0 {
return nil
}
tagEntries := []string{}
if strings.TrimSpace(tagsArg) != "" {
tagEntries = strings.Split(tagsArg, ",")
}
folderTags := make([][]string, folderCount)
// If exactly one tag entry provided, replicate it to all folders
if len(tagEntries) == 1 {
normalized := util.NormalizeTagList(strings.Split(tagEntries[0], ":"))
for i := 0; i < folderCount; i++ {
folderTags[i] = append([]string(nil), normalized...)
}
} else {
// Otherwise, assign tags to folders that have explicit entries
for i := 0; i < folderCount; i++ {
if i < len(tagEntries) {
folderTags[i] = util.NormalizeTagList(strings.Split(tagEntries[i], ":"))
} else {
// Initialize remaining folders with empty tag slice
folderTags[i] = []string{}
}
}
}
return folderTags
}
func shutdown(publicHttpDown httpdown.Server, clusterHttpServer httpdown.Server, grpcS *grpc.Server, volumeServer *weed_server.VolumeServer) {
// firstly, stop the public http service to prevent from receiving new user request
+7 -13
View File
@@ -7,28 +7,22 @@ import (
)
var cmdWorker = &Command{
UsageLine: "worker -admin=<admin_server> [-id=<worker_id>] [-jobType=all] [-workingDir=<path>] [-heartbeat=15s] [-reconnect=5s] [-maxDetect=1] [-maxExecute=4] [-metricsPort=<port>] [-metricsIp=<ip>] [-debug]",
UsageLine: "worker -admin=<admin_server> [-id=<worker_id>] [-jobType=vacuum,volume_balance,erasure_coding] [-workingDir=<path>] [-heartbeat=15s] [-reconnect=5s] [-maxDetect=1] [-maxExecute=4] [-metricsPort=<port>] [-metricsIp=<ip>] [-debug]",
Short: "start a plugin.proto worker process",
Long: `Start an external plugin worker using weed/pb/plugin.proto over gRPC.
This command provides plugin job type handlers for cluster maintenance,
including descriptor delivery, heartbeat/load reporting, detection, and execution.
This command provides vacuum, volume_balance, and erasure_coding job type
contracts with the plugin stream runtime, including descriptor delivery,
heartbeat/load reporting, detection, and execution.
Behavior:
- Use -jobType to choose handlers by category or explicit name (comma-separated)
- Categories: "all" (every registered handler), "default" (lightweight jobs),
"heavy" (resource-intensive jobs like erasure coding)
- Explicit job type names and aliases are still supported (e.g. "vacuum", "ec")
- Categories and explicit names can be mixed (e.g. "default,iceberg")
- Use -jobType to choose one or more plugin job handlers (comma-separated list)
- Use -workingDir to persist worker.id for stable worker identity across restarts
- Use -metricsPort/-metricsIp to expose /health, /ready, and /metrics
Examples:
weed worker -admin=localhost:23646
weed worker -admin=localhost:23646 -jobType=all
weed worker -admin=localhost:23646 -jobType=default
weed worker -admin=localhost:23646 -jobType=heavy
weed worker -admin=localhost:23646 -jobType=default,iceberg
weed worker -admin=localhost:23646 -jobType=volume_balance
weed worker -admin=localhost:23646 -jobType=vacuum,volume_balance
weed worker -admin=localhost:23646 -jobType=erasure_coding
weed worker -admin=admin.example.com:23646 -id=plugin-vacuum-a -heartbeat=10s
@@ -41,7 +35,7 @@ var (
workerAdminServer = cmdWorker.Flag.String("admin", "localhost:23646", "admin server address")
workerID = cmdWorker.Flag.String("id", "", "worker ID (auto-generated when empty)")
workerWorkingDir = cmdWorker.Flag.String("workingDir", "", "working directory for persistent worker state")
workerJobType = cmdWorker.Flag.String("jobType", defaultPluginWorkerJobTypes, "job types or categories to serve: all, default, heavy, or explicit names/aliases such as ec, balance, iceberg (comma-separated)")
workerJobType = cmdWorker.Flag.String("jobType", defaultPluginWorkerJobTypes, "job types to serve (comma-separated list)")
workerHeartbeat = cmdWorker.Flag.Duration("heartbeat", 15*time.Second, "heartbeat interval")
workerReconnect = cmdWorker.Flag.Duration("reconnect", 5*time.Second, "reconnect delay")
workerMaxDetect = cmdWorker.Flag.Int("maxDetect", 1, "max concurrent detection requests")
+76 -20
View File
@@ -15,7 +15,6 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/seaweedfs/seaweedfs/weed/glog"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
_ "github.com/seaweedfs/seaweedfs/weed/plugin/worker/handlers" // register all handler subpackages
"github.com/seaweedfs/seaweedfs/weed/security"
statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -24,7 +23,7 @@ import (
"google.golang.org/grpc"
)
const defaultPluginWorkerJobTypes = "all"
const defaultPluginWorkerJobTypes = "vacuum,volume_balance,erasure_coding"
type pluginWorkerRunOptions struct {
AdminServer string
@@ -138,37 +137,94 @@ func resolvePluginWorkerID(explicitID string, workingDir string) (string, error)
return worker.GenerateOrLoadWorkerID(workingDir)
}
// buildPluginWorkerHandlers resolves the comma-separated jobTypes string
// (which may contain category names like "all", "default", "heavy" and/or
// explicit job type names/aliases) into a deduplicated slice of JobHandlers.
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string) ([]pluginworker.JobHandler, error) {
jobTypes = strings.TrimSpace(jobTypes)
if jobTypes == "" {
jobTypes = defaultPluginWorkerJobTypes
}
factories, err := pluginworker.ResolveHandlerFactories(jobTypes)
// buildPluginWorkerHandler constructs the JobHandler for the given job type.
// maxExecute is forwarded to handlers that use it to report their own
// MaxExecutionConcurrency in Capability for consistency and future-proofing.
// The scheduler's effective per-worker MaxExecutionConcurrency is derived from
// the worker-level configuration (e.g. WorkerOptions.MaxExecutionConcurrency),
// not directly from the handler's Capability.
func buildPluginWorkerHandler(jobType string, dialOption grpc.DialOption, maxExecute int, workingDir string) (pluginworker.JobHandler, error) {
canonicalJobType, err := canonicalPluginWorkerJobType(jobType)
if err != nil {
return nil, err
}
opts := pluginworker.HandlerBuildOptions{
GrpcDialOption: dialOption,
MaxExecute: maxExecute,
WorkingDir: workingDir,
switch canonicalJobType {
case "vacuum":
return pluginworker.NewVacuumHandler(dialOption, int32(maxExecute)), nil
case "volume_balance":
return pluginworker.NewVolumeBalanceHandler(dialOption), nil
case "erasure_coding":
return pluginworker.NewErasureCodingHandler(dialOption, workingDir), nil
default:
return nil, fmt.Errorf("unsupported plugin job type %q", canonicalJobType)
}
}
// buildPluginWorkerHandlers constructs a deduplicated slice of JobHandlers for
// the comma-separated jobTypes string, forwarding maxExecute to each handler.
func buildPluginWorkerHandlers(jobTypes string, dialOption grpc.DialOption, maxExecute int, workingDir string) ([]pluginworker.JobHandler, error) {
parsedJobTypes, err := parsePluginWorkerJobTypes(jobTypes)
if err != nil {
return nil, err
}
handlers := make([]pluginworker.JobHandler, 0, len(factories))
for _, f := range factories {
handler, buildErr := f.Build(opts)
handlers := make([]pluginworker.JobHandler, 0, len(parsedJobTypes))
for _, jobType := range parsedJobTypes {
handler, buildErr := buildPluginWorkerHandler(jobType, dialOption, maxExecute, workingDir)
if buildErr != nil {
return nil, fmt.Errorf("building handler for %q: %w", f.JobType, buildErr)
return nil, buildErr
}
handlers = append(handlers, handler)
}
return handlers, nil
}
func parsePluginWorkerJobTypes(jobTypes string) ([]string, error) {
jobTypes = strings.TrimSpace(jobTypes)
if jobTypes == "" {
return []string{"vacuum"}, nil
}
parts := strings.Split(jobTypes, ",")
parsed := make([]string, 0, len(parts))
seen := make(map[string]struct{}, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "" {
continue
}
canonical, err := canonicalPluginWorkerJobType(part)
if err != nil {
return nil, err
}
if _, found := seen[canonical]; found {
continue
}
seen[canonical] = struct{}{}
parsed = append(parsed, canonical)
}
if len(parsed) == 0 {
return []string{"vacuum"}, nil
}
return parsed, nil
}
func canonicalPluginWorkerJobType(jobType string) (string, error) {
switch strings.ToLower(strings.TrimSpace(jobType)) {
case "", "vacuum":
return "vacuum", nil
case "volume_balance", "balance", "volume.balance", "volume-balance":
return "volume_balance", nil
case "erasure_coding", "erasure-coding", "erasure.coding", "ec":
return "erasure_coding", nil
default:
return "", fmt.Errorf("unsupported plugin job type %q", jobType)
}
}
func resolvePluginWorkerAdminServer(adminServer string) string {
adminServer = strings.TrimSpace(adminServer)
host, httpPort, hasExplicitGrpcPort, err := parsePluginWorkerAdminAddress(adminServer)
+5 -13
View File
@@ -1,21 +1,13 @@
package command
import (
"testing"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
import "testing"
func TestWorkerDefaultJobTypes(t *testing.T) {
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
handlers, err := buildPluginWorkerHandlers(*workerJobType, dialOption, int(pluginworker.DefaultMaxExecutionConcurrency), "")
jobTypes, err := parsePluginWorkerJobTypes(*workerJobType)
if err != nil {
t.Fatalf("buildPluginWorkerHandlers(default worker flag) err = %v", err)
t.Fatalf("parsePluginWorkerJobTypes(default worker flag) err = %v", err)
}
// Expected: vacuum, volume_balance, admin_script, erasure_coding, iceberg_maintenance
if len(handlers) != 5 {
t.Fatalf("expected default worker job types to include 5 handlers, got %d", len(handlers))
if len(jobTypes) != 3 {
t.Fatalf("expected default worker job types to include 3 handlers, got %v", jobTypes)
}
}
@@ -24,7 +24,7 @@ func (store *FilerEtcStore) LoadConfiguration(ctx context.Context) (*iam_pb.S3Ap
s3cfg := &iam_pb.S3ApiConfiguration{}
// 1. Load from legacy single file (low priority)
content, foundLegacy, err := store.readInsideFiler(ctx, filer.IamConfigDirectory, IamLegacyIdentityFile)
content, foundLegacy, err := store.readInsideFiler(filer.IamConfigDirectory, IamLegacyIdentityFile)
if err != nil {
return s3cfg, err
}
@@ -93,7 +93,7 @@ func (store *FilerEtcStore) loadFromMultiFile(ctx context.Context, s3cfg *iam_pb
if len(entry.Content) > 0 {
content = entry.Content
} else {
c, err := filer.ReadInsideFiler(ctx, client, dir, entry.Name)
c, err := filer.ReadInsideFiler(client, dir, entry.Name)
if err != nil {
glog.Warningf("Failed to read identity file %s: %v", entry.Name, err)
continue
@@ -249,7 +249,7 @@ func (store *FilerEtcStore) CreateUser(ctx context.Context, identity *iam_pb.Ide
func (store *FilerEtcStore) GetUser(ctx context.Context, username string) (*iam_pb.Identity, error) {
var identity *iam_pb.Identity
err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
data, err := filer.ReadInsideFiler(ctx, client, filer.IamConfigDirectory+"/"+IamIdentitiesDirectory, username+".json")
data, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory+"/"+IamIdentitiesDirectory, username+".json")
if err != nil {
if err == filer_pb.ErrNotFound {
return credential.ErrUserNotFound
@@ -350,7 +350,7 @@ func (store *FilerEtcStore) GetUserByAccessKey(ctx context.Context, accessKey st
if len(entry.Content) > 0 {
content = entry.Content
} else {
c, err := filer.ReadInsideFiler(ctx, client, dir, entry.Name)
c, err := filer.ReadInsideFiler(client, dir, entry.Name)
if err != nil {
continue
}
@@ -435,11 +435,11 @@ func (store *FilerEtcStore) saveIdentity(ctx context.Context, identity *iam_pb.I
})
}
func (store *FilerEtcStore) readInsideFiler(ctx context.Context, dir string, name string) ([]byte, bool, error) {
func (store *FilerEtcStore) readInsideFiler(dir string, name string) ([]byte, bool, error) {
var content []byte
found := false
err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
c, err := filer.ReadInsideFiler(ctx, client, dir, name)
c, err := filer.ReadInsideFiler(client, dir, name)
if err != nil {
if err == filer_pb.ErrNotFound {
return nil
+29 -152
View File
@@ -9,7 +9,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
)
@@ -19,113 +18,13 @@ const (
)
type PoliciesCollection struct {
Policies map[string]policy_engine.PolicyDocument `json:"policies"`
InlinePolicies map[string]map[string]policy_engine.PolicyDocument `json:"inlinePolicies"`
Policies map[string]policy_engine.PolicyDocument `json:"policies"`
}
func validatePolicyName(name string) error {
return credential.ValidatePolicyName(name)
}
func newPoliciesCollection() *PoliciesCollection {
return &PoliciesCollection{
Policies: make(map[string]policy_engine.PolicyDocument),
InlinePolicies: make(map[string]map[string]policy_engine.PolicyDocument),
}
}
func (store *FilerEtcStore) loadLegacyPoliciesCollection(ctx context.Context) (*PoliciesCollection, bool, error) {
policiesCollection := newPoliciesCollection()
content, foundLegacy, err := store.readInsideFiler(ctx, filer.IamConfigDirectory, filer.IamPoliciesFile)
if err != nil {
return nil, false, err
}
if !foundLegacy || len(content) == 0 {
return policiesCollection, foundLegacy, nil
}
if err := json.Unmarshal(content, policiesCollection); err != nil {
return nil, false, err
}
if policiesCollection.Policies == nil {
policiesCollection.Policies = make(map[string]policy_engine.PolicyDocument)
}
if policiesCollection.InlinePolicies == nil {
policiesCollection.InlinePolicies = make(map[string]map[string]policy_engine.PolicyDocument)
}
return policiesCollection, true, nil
}
func (store *FilerEtcStore) saveLegacyPoliciesCollection(ctx context.Context, policiesCollection *PoliciesCollection) error {
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
content, err := json.MarshalIndent(policiesCollection, "", " ")
if err != nil {
return err
}
return filer.SaveInsideFiler(client, filer.IamConfigDirectory, filer.IamPoliciesFile, content)
})
}
func policyDocumentToPbPolicy(name string, policy policy_engine.PolicyDocument) (*iam_pb.Policy, error) {
content, err := json.Marshal(policy)
if err != nil {
return nil, err
}
return &iam_pb.Policy{Name: name, Content: string(content)}, nil
}
// LoadManagedPolicies loads managed policies for the S3 runtime without
// triggering legacy-to-multifile migration. This lets the runtime hydrate
// policies while preserving any legacy inline policy data stored alongside
// managed policies.
func (store *FilerEtcStore) LoadManagedPolicies(ctx context.Context) ([]*iam_pb.Policy, error) {
policiesCollection, _, err := store.loadLegacyPoliciesCollection(ctx)
if err != nil {
return nil, err
}
policies := make(map[string]policy_engine.PolicyDocument, len(policiesCollection.Policies))
for name, policy := range policiesCollection.Policies {
policies[name] = policy
}
if err := store.loadPoliciesFromMultiFile(ctx, policies); err != nil {
return nil, err
}
managedPolicies := make([]*iam_pb.Policy, 0, len(policies))
for name, policy := range policies {
pbPolicy, err := policyDocumentToPbPolicy(name, policy)
if err != nil {
return nil, err
}
managedPolicies = append(managedPolicies, pbPolicy)
}
return managedPolicies, nil
}
// LoadInlinePolicies loads legacy inline policies keyed by user name. Inline
// policies are still stored in the legacy shared policies file.
func (store *FilerEtcStore) LoadInlinePolicies(ctx context.Context) (map[string]map[string]policy_engine.PolicyDocument, error) {
policiesCollection, _, err := store.loadLegacyPoliciesCollection(ctx)
if err != nil {
return nil, err
}
inlinePolicies := make(map[string]map[string]policy_engine.PolicyDocument, len(policiesCollection.InlinePolicies))
for userName, userPolicies := range policiesCollection.InlinePolicies {
inlinePolicies[userName] = make(map[string]policy_engine.PolicyDocument, len(userPolicies))
for policyName, policy := range userPolicies {
inlinePolicies[userName][policyName] = policy
}
}
return inlinePolicies, nil
}
// GetPolicies retrieves all IAM policies from the filer
func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_engine.PolicyDocument, error) {
policies := make(map[string]policy_engine.PolicyDocument)
@@ -144,12 +43,23 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_
filer.IamConfigDirectory, filer.IamPoliciesFile)
// 1. Load from legacy single file (low priority)
policiesCollection, _, err := store.loadLegacyPoliciesCollection(ctx)
content, foundLegacy, err := store.readInsideFiler(filer.IamConfigDirectory, filer.IamPoliciesFile)
if err != nil {
return nil, err
}
for name, policy := range policiesCollection.Policies {
policies[name] = policy
if foundLegacy && len(content) > 0 {
policiesCollection := &PoliciesCollection{
Policies: make(map[string]policy_engine.PolicyDocument),
}
if err := json.Unmarshal(content, policiesCollection); err != nil {
glog.Errorf("Failed to parse legacy IAM policies from %s/%s: %v",
filer.IamConfigDirectory, filer.IamPoliciesFile, err)
} else {
for name, policy := range policiesCollection.Policies {
policies[name] = policy
}
}
}
// 2. Load from multi-file structure (high priority, overrides legacy)
@@ -157,6 +67,14 @@ func (store *FilerEtcStore) GetPolicies(ctx context.Context) (map[string]policy_
return nil, err
}
// 3. Perform migration if we loaded legacy config
if foundLegacy {
if err := store.migratePoliciesToMultiFile(ctx, policies); err != nil {
glog.Errorf("Failed to migrate IAM policies to multi-file layout: %v", err)
return policies, err
}
}
return policies, nil
}
@@ -180,7 +98,7 @@ func (store *FilerEtcStore) loadPoliciesFromMultiFile(ctx context.Context, polic
if len(entry.Content) > 0 {
content = entry.Content
} else {
c, err := filer.ReadInsideFiler(ctx, client, dir, entry.Name)
c, err := filer.ReadInsideFiler(client, dir, entry.Name)
if err != nil {
glog.Warningf("Failed to read policy file %s: %v", entry.Name, err)
continue
@@ -197,7 +115,7 @@ func (store *FilerEtcStore) loadPoliciesFromMultiFile(ctx context.Context, polic
// The file name is "policyName.json"
policyName := entry.Name
if strings.HasSuffix(policyName, ".json") {
if len(policyName) > 5 && policyName[len(policyName)-5:] == ".json" {
policyName = policyName[:len(policyName)-5]
policies[policyName] = policy
}
@@ -266,23 +184,7 @@ func (store *FilerEtcStore) DeletePolicy(ctx context.Context, name string) error
if err := validatePolicyName(name); err != nil {
return err
}
store.policyMu.Lock()
defer store.policyMu.Unlock()
policiesCollection, foundLegacy, err := store.loadLegacyPoliciesCollection(ctx)
if err != nil {
return err
}
deleteLegacyPolicy := false
if foundLegacy {
if _, exists := policiesCollection.Policies[name]; exists {
delete(policiesCollection.Policies, name)
deleteLegacyPolicy = true
}
}
if err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
return store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
_, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
Directory: filer.IamConfigDirectory + "/" + IamPoliciesDirectory,
Name: name + ".json",
@@ -291,15 +193,7 @@ func (store *FilerEtcStore) DeletePolicy(ctx context.Context, name string) error
return err
}
return nil
}); err != nil {
return err
}
if deleteLegacyPolicy {
return store.saveLegacyPoliciesCollection(ctx, policiesCollection)
}
return nil
})
}
// GetPolicy retrieves a specific IAM policy by name from the filer
@@ -310,7 +204,7 @@ func (store *FilerEtcStore) GetPolicy(ctx context.Context, name string) (*policy
var policy *policy_engine.PolicyDocument
err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
data, err := filer.ReadInsideFiler(ctx, client, filer.IamConfigDirectory+"/"+IamPoliciesDirectory, name+".json")
data, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory+"/"+IamPoliciesDirectory, name+".json")
if err != nil {
if err == filer_pb.ErrNotFound {
return nil
@@ -345,7 +239,6 @@ func (store *FilerEtcStore) GetPolicy(ctx context.Context, name string) (*policy
// ListPolicyNames returns all managed policy names stored in the filer.
func (store *FilerEtcStore) ListPolicyNames(ctx context.Context) ([]string, error) {
names := make([]string, 0)
seenNames := make(map[string]struct{})
store.mu.RLock()
configured := store.filerAddressFunc != nil
@@ -355,19 +248,7 @@ func (store *FilerEtcStore) ListPolicyNames(ctx context.Context) ([]string, erro
return names, nil
}
policiesCollection, _, err := store.loadLegacyPoliciesCollection(ctx)
if err != nil {
return nil, err
}
for name := range policiesCollection.Policies {
if _, found := seenNames[name]; found {
continue
}
names = append(names, name)
seenNames[name] = struct{}{}
}
err = store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
dir := filer.IamConfigDirectory + "/" + IamPoliciesDirectory
entries, err := listEntries(ctx, client, dir)
if err != nil {
@@ -385,11 +266,7 @@ func (store *FilerEtcStore) ListPolicyNames(ctx context.Context) ([]string, erro
if strings.HasSuffix(name, ".json") {
name = name[:len(name)-5]
}
if _, found := seenNames[name]; found {
continue
}
names = append(names, name)
seenNames[name] = struct{}{}
}
return nil
@@ -1,369 +0,0 @@
package filer_etc
import (
"context"
"net"
"sort"
"strconv"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/iam_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
)
type policyTestFilerServer struct {
filer_pb.UnimplementedSeaweedFilerServer
mu sync.RWMutex
entries map[string]*filer_pb.Entry
contentlessListEntry map[string]struct{}
beforeLookup func(context.Context, string, string) error
afterListEntry func(string, string)
beforeDelete func(string, string) error
beforeUpdate func(string, string) error
}
func newPolicyTestFilerServer() *policyTestFilerServer {
return &policyTestFilerServer{
entries: make(map[string]*filer_pb.Entry),
contentlessListEntry: make(map[string]struct{}),
}
}
func (s *policyTestFilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
s.mu.RLock()
beforeLookup := s.beforeLookup
s.mu.RUnlock()
if beforeLookup != nil {
if err := beforeLookup(ctx, req.Directory, req.Name); err != nil {
return nil, err
}
}
s.mu.RLock()
defer s.mu.RUnlock()
entry, found := s.entries[filerEntryKey(req.Directory, req.Name)]
if !found {
return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
}
return &filer_pb.LookupDirectoryEntryResponse{Entry: cloneEntry(entry)}, nil
}
func (s *policyTestFilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream grpc.ServerStreamingServer[filer_pb.ListEntriesResponse]) error {
s.mu.RLock()
defer s.mu.RUnlock()
names := make([]string, 0)
for key := range s.entries {
dir, name := splitFilerEntryKey(key)
if dir != req.Directory {
continue
}
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
entry := cloneEntry(s.entries[filerEntryKey(req.Directory, name)])
if _, found := s.contentlessListEntry[filerEntryKey(req.Directory, name)]; found {
entry.Content = nil
}
if err := stream.Send(&filer_pb.ListEntriesResponse{Entry: entry}); err != nil {
return err
}
if s.afterListEntry != nil {
s.afterListEntry(req.Directory, name)
}
}
return nil
}
func (s *policyTestFilerServer) CreateEntry(_ context.Context, req *filer_pb.CreateEntryRequest) (*filer_pb.CreateEntryResponse, error) {
s.mu.Lock()
defer s.mu.Unlock()
s.entries[filerEntryKey(req.Directory, req.Entry.Name)] = cloneEntry(req.Entry)
return &filer_pb.CreateEntryResponse{}, nil
}
func (s *policyTestFilerServer) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
s.mu.RLock()
beforeUpdate := s.beforeUpdate
s.mu.RUnlock()
if beforeUpdate != nil {
if err := beforeUpdate(req.Directory, req.Entry.Name); err != nil {
return nil, err
}
}
s.mu.Lock()
defer s.mu.Unlock()
s.entries[filerEntryKey(req.Directory, req.Entry.Name)] = cloneEntry(req.Entry)
return &filer_pb.UpdateEntryResponse{}, nil
}
func (s *policyTestFilerServer) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
s.mu.RLock()
beforeDelete := s.beforeDelete
s.mu.RUnlock()
if beforeDelete != nil {
if err := beforeDelete(req.Directory, req.Name); err != nil {
return nil, err
}
}
s.mu.Lock()
defer s.mu.Unlock()
key := filerEntryKey(req.Directory, req.Name)
if _, found := s.entries[key]; !found {
return nil, status.Error(codes.NotFound, filer_pb.ErrNotFound.Error())
}
delete(s.entries, key)
return &filer_pb.DeleteEntryResponse{}, nil
}
func newPolicyTestStore(t *testing.T) *FilerEtcStore {
store, _ := newPolicyTestStoreWithServer(t)
return store
}
func newPolicyTestStoreWithServer(t *testing.T) (*FilerEtcStore, *policyTestFilerServer) {
t.Helper()
lis, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
server := newPolicyTestFilerServer()
grpcServer := pb.NewGrpcServer()
filer_pb.RegisterSeaweedFilerServer(grpcServer, server)
go func() {
_ = grpcServer.Serve(lis)
}()
t.Cleanup(func() {
grpcServer.Stop()
_ = lis.Close()
})
store := &FilerEtcStore{}
host, portString, err := net.SplitHostPort(lis.Addr().String())
require.NoError(t, err)
grpcPort, err := strconv.Atoi(portString)
require.NoError(t, err)
store.SetFilerAddressFunc(func() pb.ServerAddress {
return pb.NewServerAddress(host, 1, grpcPort)
}, grpc.WithTransportCredentials(insecure.NewCredentials()))
return store, server
}
func TestFilerEtcStoreListPolicyNamesIncludesLegacyPolicies(t *testing.T) {
ctx := context.Background()
store := newPolicyTestStore(t)
legacyPolicies := newPoliciesCollection()
legacyPolicies.Policies["legacy-only"] = testPolicyDocument("s3:GetObject", "arn:aws:s3:::legacy-only/*")
legacyPolicies.Policies["shared"] = testPolicyDocument("s3:GetObject", "arn:aws:s3:::shared/*")
require.NoError(t, store.saveLegacyPoliciesCollection(ctx, legacyPolicies))
require.NoError(t, store.savePolicy(ctx, "multi-file-only", testPolicyDocument("s3:PutObject", "arn:aws:s3:::multi-file-only/*")))
require.NoError(t, store.savePolicy(ctx, "shared", testPolicyDocument("s3:DeleteObject", "arn:aws:s3:::shared/*")))
names, err := store.ListPolicyNames(ctx)
require.NoError(t, err)
assert.ElementsMatch(t, []string{"legacy-only", "multi-file-only", "shared"}, names)
}
func TestFilerEtcStoreDeletePolicyRemovesLegacyManagedCopy(t *testing.T) {
ctx := context.Background()
store := newPolicyTestStore(t)
inlinePolicy := testPolicyDocument("s3:PutObject", "arn:aws:s3:::inline-user/*")
legacyPolicies := newPoliciesCollection()
legacyPolicies.Policies["legacy-only"] = testPolicyDocument("s3:GetObject", "arn:aws:s3:::legacy-only/*")
legacyPolicies.InlinePolicies["inline-user"] = map[string]policy_engine.PolicyDocument{
"PutOnly": inlinePolicy,
}
require.NoError(t, store.saveLegacyPoliciesCollection(ctx, legacyPolicies))
managedPolicies, err := store.LoadManagedPolicies(ctx)
require.NoError(t, err)
assert.Equal(t, []string{"legacy-only"}, managedPolicyNames(managedPolicies))
require.NoError(t, store.DeletePolicy(ctx, "legacy-only"))
managedPolicies, err = store.LoadManagedPolicies(ctx)
require.NoError(t, err)
assert.Empty(t, managedPolicies)
inlinePolicies, err := store.LoadInlinePolicies(ctx)
require.NoError(t, err)
assertInlinePolicyPreserved(t, inlinePolicies, "inline-user", "PutOnly")
loadedLegacyPolicies, foundLegacy, err := store.loadLegacyPoliciesCollection(ctx)
require.NoError(t, err)
require.True(t, foundLegacy)
assert.Empty(t, loadedLegacyPolicies.Policies)
assertInlinePolicyPreserved(t, loadedLegacyPolicies.InlinePolicies, "inline-user", "PutOnly")
}
func TestFilerEtcStoreDeletePolicySerializesLegacyUpdates(t *testing.T) {
ctx := context.Background()
store, server := newPolicyTestStoreWithServer(t)
legacyPolicies := newPoliciesCollection()
legacyPolicies.Policies["first"] = testPolicyDocument("s3:GetObject", "arn:aws:s3:::first/*")
legacyPolicies.Policies["second"] = testPolicyDocument("s3:GetObject", "arn:aws:s3:::second/*")
require.NoError(t, store.saveLegacyPoliciesCollection(ctx, legacyPolicies))
require.NoError(t, store.savePolicy(ctx, "first", testPolicyDocument("s3:GetObject", "arn:aws:s3:::first/*")))
require.NoError(t, store.savePolicy(ctx, "second", testPolicyDocument("s3:GetObject", "arn:aws:s3:::second/*")))
firstSaveStarted := make(chan struct{})
releaseFirstSave := make(chan struct{})
secondReachedDelete := make(chan struct{}, 1)
var blockOnce sync.Once
server.mu.Lock()
server.beforeUpdate = func(dir string, name string) error {
if dir == filer.IamConfigDirectory && name == filer.IamPoliciesFile {
blockOnce.Do(func() {
close(firstSaveStarted)
<-releaseFirstSave
})
}
return nil
}
server.beforeDelete = func(dir string, name string) error {
if dir == filer.IamConfigDirectory+"/"+IamPoliciesDirectory && name == "second.json" {
select {
case secondReachedDelete <- struct{}{}:
default:
}
}
return nil
}
server.mu.Unlock()
firstDeleteErr := make(chan error, 1)
go func() {
firstDeleteErr <- store.DeletePolicy(ctx, "first")
}()
<-firstSaveStarted
secondDeleteErr := make(chan error, 1)
go func() {
secondDeleteErr <- store.DeletePolicy(ctx, "second")
}()
select {
case <-secondReachedDelete:
t.Fatal("second delete reached filer mutation while first delete was still blocked")
case <-time.After(300 * time.Millisecond):
}
close(releaseFirstSave)
require.NoError(t, <-firstDeleteErr)
require.NoError(t, <-secondDeleteErr)
loadedLegacyPolicies, foundLegacy, err := store.loadLegacyPoliciesCollection(ctx)
require.NoError(t, err)
require.True(t, foundLegacy)
assert.Empty(t, loadedLegacyPolicies.Policies)
}
func TestFilerEtcStoreLoadManagedPoliciesRespectsReadContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
store, server := newPolicyTestStoreWithServer(t)
require.NoError(t, store.savePolicy(context.Background(), "cancel-me", testPolicyDocument("s3:GetObject", "arn:aws:s3:::cancel-me/*")))
server.mu.Lock()
server.contentlessListEntry[filerEntryKey(filer.IamConfigDirectory+"/"+IamPoliciesDirectory, "cancel-me.json")] = struct{}{}
server.beforeLookup = func(ctx context.Context, dir string, name string) error {
if dir == filer.IamConfigDirectory+"/"+IamPoliciesDirectory && name == "cancel-me.json" {
cancel()
return status.Error(codes.Canceled, context.Canceled.Error())
}
return nil
}
server.mu.Unlock()
managedPolicies, err := store.LoadManagedPolicies(ctx)
require.NoError(t, err)
assert.Empty(t, managedPolicies)
}
func testPolicyDocument(action string, resource string) policy_engine.PolicyDocument {
return policy_engine.PolicyDocument{
Version: policy_engine.PolicyVersion2012_10_17,
Statement: []policy_engine.PolicyStatement{
{
Effect: policy_engine.PolicyEffectAllow,
Action: policy_engine.NewStringOrStringSlice(action),
Resource: policy_engine.NewStringOrStringSlice(resource),
},
},
}
}
func managedPolicyNames(policies []*iam_pb.Policy) []string {
names := make([]string, 0, len(policies))
for _, policy := range policies {
names = append(names, policy.Name)
}
sort.Strings(names)
return names
}
func assertInlinePolicyPreserved(t *testing.T, inlinePolicies map[string]map[string]policy_engine.PolicyDocument, userName string, policyName string) {
t.Helper()
userPolicies, found := inlinePolicies[userName]
require.True(t, found)
policy, found := userPolicies[policyName]
require.True(t, found)
assert.Equal(t, policy_engine.PolicyVersion2012_10_17, policy.Version)
require.Len(t, policy.Statement, 1)
assert.Equal(t, policy_engine.PolicyEffectAllow, policy.Statement[0].Effect)
}
func cloneEntry(entry *filer_pb.Entry) *filer_pb.Entry {
if entry == nil {
return nil
}
return proto.Clone(entry).(*filer_pb.Entry)
}
func filerEntryKey(dir string, name string) string {
return dir + "\x00" + name
}
func splitFilerEntryKey(key string) (dir string, name string) {
for idx := 0; idx < len(key); idx++ {
if key[idx] == '\x00' {
return key[:idx], key[idx+1:]
}
}
return key, ""
}
@@ -38,7 +38,7 @@ func (store *FilerEtcStore) loadServiceAccountsFromMultiFile(ctx context.Context
if len(entry.Content) > 0 {
content = entry.Content
} else {
c, err := filer.ReadInsideFiler(ctx, client, dir, entry.Name)
c, err := filer.ReadInsideFiler(client, dir, entry.Name)
if err != nil {
glog.Warningf("Failed to read service account file %s: %v", entry.Name, err)
continue
@@ -133,7 +133,7 @@ func (store *FilerEtcStore) GetServiceAccount(ctx context.Context, id string) (*
}
var sa *iam_pb.ServiceAccount
err := store.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
data, err := filer.ReadInsideFiler(ctx, client, filer.IamConfigDirectory+"/"+IamServiceAccountsDirectory, id+".json")
data, err := filer.ReadInsideFiler(client, filer.IamConfigDirectory+"/"+IamServiceAccountsDirectory, id+".json")
if err != nil {
if err == filer_pb.ErrNotFound {
return credential.ErrServiceAccountNotFound
@@ -170,7 +170,7 @@ func (store *FilerEtcStore) ListServiceAccounts(ctx context.Context) ([]*iam_pb.
if len(entry.Content) > 0 {
content = entry.Content
} else {
c, err := filer.ReadInsideFiler(ctx, client, dir, entry.Name)
c, err := filer.ReadInsideFiler(client, dir, entry.Name)
if err != nil {
glog.Warningf("Failed to read service account file %s: %v", entry.Name, err)
continue
@@ -20,7 +20,6 @@ type FilerEtcStore struct {
filerAddressFunc func() pb.ServerAddress // Function to get current active filer
grpcDialOption grpc.DialOption
mu sync.RWMutex // Protects filerAddressFunc and grpcDialOption
policyMu sync.Mutex // Serializes legacy managed-policy mutations
}
func (store *FilerEtcStore) GetName() credential.CredentialStoreTypeName {
-40
View File
@@ -20,14 +20,6 @@ import (
var _ CredentialStore = &PropagatingCredentialStore{}
var _ PolicyManager = &PropagatingCredentialStore{}
type propagatingManagedPolicyLoader interface {
LoadManagedPolicies(ctx context.Context) ([]*iam_pb.Policy, error)
}
type propagatingInlinePolicyLoader interface {
LoadInlinePolicies(ctx context.Context) (map[string]map[string]policy_engine.PolicyDocument, error)
}
type PropagatingCredentialStore struct {
CredentialStore
masterClient *wdclient.MasterClient
@@ -248,38 +240,6 @@ func (s *PropagatingCredentialStore) ListPolicyNames(ctx context.Context) ([]str
return s.CredentialStore.ListPolicyNames(ctx)
}
func (s *PropagatingCredentialStore) LoadManagedPolicies(ctx context.Context) ([]*iam_pb.Policy, error) {
if loader, ok := s.CredentialStore.(propagatingManagedPolicyLoader); ok {
return loader.LoadManagedPolicies(ctx)
}
policies, err := s.CredentialStore.GetPolicies(ctx)
if err != nil {
return nil, err
}
managedPolicies := make([]*iam_pb.Policy, 0, len(policies))
for name, policyDocument := range policies {
content, err := json.Marshal(policyDocument)
if err != nil {
return nil, err
}
managedPolicies = append(managedPolicies, &iam_pb.Policy{
Name: name,
Content: string(content),
})
}
return managedPolicies, nil
}
func (s *PropagatingCredentialStore) LoadInlinePolicies(ctx context.Context) (map[string]map[string]policy_engine.PolicyDocument, error) {
if loader, ok := s.CredentialStore.(propagatingInlinePolicyLoader); ok {
return loader.LoadInlinePolicies(ctx)
}
return nil, nil
}
func (s *PropagatingCredentialStore) CreatePolicy(ctx context.Context, name string, document policy_engine.PolicyDocument) error {
if pm, ok := s.CredentialStore.(PolicyManager); ok {
if err := pm.CreatePolicy(ctx, name, document); err != nil {
-10
View File
@@ -25,7 +25,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"golang.org/x/sync/singleflight"
)
const (
@@ -55,7 +54,6 @@ type Filer struct {
Signature int32
FilerConf *FilerConf
RemoteStorage *FilerRemoteStorage
lazyFetchGroup singleflight.Group
Dlm *lock_manager.DistributedLockManager
MaxFilenameLength uint32
deletionQuit chan struct{}
@@ -377,14 +375,6 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e
}
}
if entry == nil && (err == nil || errors.Is(err, filer_pb.ErrNotFound)) {
if lazy, lazyErr := f.maybeLazyFetchFromRemote(ctx, p); lazyErr != nil {
glog.V(1).InfofCtx(ctx, "FindEntry lazy fetch %s: %v", p, lazyErr)
} else if lazy != nil {
return lazy, nil
}
}
return entry, err
}
+1 -1
View File
@@ -47,7 +47,7 @@ func ReadFilerConfFromFilers(filerGrpcAddresses []pb.ServerAddress, grpcDialOpti
data = buf.Bytes()
return nil
}
content, err := ReadInsideFiler(context.Background(), client, DirectoryEtcSeaweedFS, FilerConfName)
content, err := ReadInsideFiler(client, DirectoryEtcSeaweedFS, FilerConfName)
if err != nil {
return err
}
-116
View File
@@ -1,116 +0,0 @@
package filer
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type lazyFetchContextKey struct{}
// maybeLazyFetchFromRemote is called by FindEntry when the store returns no
// entry for p. If p is under a remote-storage mount, it stats the remote
// object, builds a filer Entry from the result, and persists it via
// CreateEntry with SkipCheckParentDirectory so phantom parent directories
// under the mount are not required.
//
// On a CreateEntry failure after a successful StatFile the in-memory entry is
// still returned (availability over consistency); the singleflight key is
// forgotten so the next lookup retries the filer write.
//
// Returns nil without error when: p is not under a remote mount; the remote
// reports the object does not exist; or any other remote error occurs.
func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (*Entry, error) {
// Prevent recursive invocation: CreateEntry calls FindEntry, which would
// re-enter this function and deadlock on the singleflight key.
if ctx.Value(lazyFetchContextKey{}) != nil {
return nil, nil
}
if f.RemoteStorage == nil {
return nil, nil
}
mountDir, remoteLoc := f.RemoteStorage.FindMountDirectory(p)
if remoteLoc == nil {
return nil, nil
}
client, _, found := f.RemoteStorage.FindRemoteStorageClient(p)
if !found {
return nil, nil
}
relPath := strings.TrimPrefix(string(p), string(mountDir))
if relPath != "" && !strings.HasPrefix(relPath, "/") {
relPath = "/" + relPath
}
base := strings.TrimSuffix(remoteLoc.Path, "/")
remotePath := "/" + strings.TrimLeft(base+relPath, "/")
objectLoc := &remote_pb.RemoteStorageLocation{
Name: remoteLoc.Name,
Bucket: remoteLoc.Bucket,
Path: remotePath,
}
type lazyFetchResult struct {
entry *Entry
}
key := string(p)
val, err, _ := f.lazyFetchGroup.Do(key, func() (interface{}, error) {
remoteEntry, statErr := client.StatFile(objectLoc)
if statErr != nil {
if errors.Is(statErr, remote_storage.ErrRemoteObjectNotFound) {
glog.V(3).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s not found in remote", p)
} else {
glog.Warningf("maybeLazyFetchFromRemote: stat %s failed: %v", p, statErr)
}
return lazyFetchResult{nil}, nil
}
if remoteEntry == nil {
glog.V(3).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s StatFile returned nil entry", p)
return lazyFetchResult{nil}, nil
}
mtime := time.Unix(remoteEntry.RemoteMtime, 0)
entry := &Entry{
FullPath: p,
Attr: Attr{
Mtime: mtime,
Crtime: mtime,
Mode: 0644,
FileSize: uint64(remoteEntry.RemoteSize),
},
Remote: remoteEntry,
}
persistBaseCtx, cancelPersist := context.WithTimeout(context.Background(), 30*time.Second)
defer cancelPersist()
persistCtx := context.WithValue(persistBaseCtx, lazyFetchContextKey{}, true)
saveErr := f.CreateEntry(persistCtx, entry, false, false, nil, true, f.MaxFilenameLength)
if saveErr != nil {
glog.Warningf("maybeLazyFetchFromRemote: failed to persist filer entry for %s: %v", p, saveErr)
f.lazyFetchGroup.Forget(key)
}
return lazyFetchResult{entry}, nil
})
if err != nil {
return nil, err
}
result, ok := val.(lazyFetchResult)
if !ok {
return nil, fmt.Errorf("maybeLazyFetchFromRemote: unexpected singleflight result type %T for %s", val, p)
}
return result.entry, nil
}

Some files were not shown because too many files have changed in this diff Show More