Compare commits

..
95 changed files with 1056 additions and 6471 deletions
+1 -1
View File
@@ -135,7 +135,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: output-logs
path: docker/output.log
+1 -1
View File
@@ -52,7 +52,7 @@ jobs:
- name: Archive logs
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ec-integration-test-logs
path: |
+1 -1
View File
@@ -42,7 +42,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ec-test-logs
path: test/erasure_coding/admin_dockertest/tmp/logs/
+1 -1
View File
@@ -183,7 +183,7 @@ jobs:
- name: Upload Test Artifacts
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: fuse-integration-test-results
path: |
+1 -1
View File
@@ -49,7 +49,7 @@ jobs:
- name: Upload Test Reports
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: test-reports-java-${{ matrix.java }}
path: |
@@ -70,7 +70,7 @@ jobs:
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: metadata-subscribe-test-logs
path: |
+1 -1
View File
@@ -62,7 +62,7 @@ jobs:
- name: Archive logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: postgres-logs
path: test/postgres/postgres-output.log
@@ -57,7 +57,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: integration-test-logs
path: test/s3/normal/*.log
+1 -1
View File
@@ -77,7 +77,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-filer-group-test-logs
path: test/s3/filer_group/weed-test*.log
+9 -9
View File
@@ -76,7 +76,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-test-logs-${{ matrix.test-type }}
path: test/s3/versioning/weed-test*.log
@@ -124,7 +124,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-compatibility-logs
path: test/s3/versioning/weed-test*.log
@@ -172,7 +172,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-cors-compatibility-logs
path: test/s3/cors/weed-test*.log
@@ -239,7 +239,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-retention-test-logs-${{ matrix.test-type }}
path: test/s3/retention/weed-test*.log
@@ -306,7 +306,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-cors-test-logs-${{ matrix.test-type }}
path: test/s3/cors/weed-test*.log
@@ -355,7 +355,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-retention-worm-logs
path: test/s3/retention/weed-test*.log
@@ -422,7 +422,7 @@ jobs:
- name: Upload stress test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-versioning-stress-logs
path: test/s3/versioning/weed-test*.log
@@ -478,7 +478,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-tagging-test-logs
path: test/s3/tagging/weed-test*.log
@@ -531,7 +531,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-remote-cache-test-logs
path: |
+4 -4
View File
@@ -65,7 +65,7 @@ jobs:
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: iam-unit-test-results
path: |
@@ -162,7 +162,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-integration-logs-${{ matrix.test-type }}
path: test/s3/iam/weed-*.log
@@ -222,7 +222,7 @@ jobs:
- name: Upload distributed test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-distributed-logs
path: test/s3/iam/weed-*.log
@@ -274,7 +274,7 @@ jobs:
- name: Upload performance test results
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-iam-performance-results
path: |
+1 -1
View File
@@ -152,7 +152,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-keycloak-test-logs
path: |
+1 -1
View File
@@ -121,7 +121,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: test-logs-python-${{ matrix.python-version }}
path: |
+4 -4
View File
@@ -70,7 +70,7 @@ jobs:
- name: Upload test results on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: policy-unit-test-results
path: |
@@ -178,7 +178,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-policy-variables-test-logs
path: /tmp/weed_policy_test_server.log
@@ -299,7 +299,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-policy-enforcement-logs-${{ matrix.test-case }}
path: /tmp/weed_policy_enforcement_${{ matrix.test-case }}.log
@@ -386,7 +386,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: trusted-proxy-test-logs
path: /tmp/weed_proxy_test.log
+1 -1
View File
@@ -73,7 +73,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-spark-test-logs
path: test/s3/spark/test-output.log
+7 -7
View File
@@ -95,7 +95,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-test-logs-${{ matrix.test-type }}
path: test/s3/sse/weed-test*.log
@@ -143,7 +143,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-compatibility-logs
path: test/s3/sse/weed-test*.log
@@ -192,7 +192,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-metadata-persistence-logs
path: test/s3/sse/weed-test*.log
@@ -241,7 +241,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-copy-operations-logs
path: test/s3/sse/weed-test*.log
@@ -290,7 +290,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-multipart-logs
path: test/s3/sse/weed-test*.log
@@ -340,7 +340,7 @@ jobs:
- name: Upload performance test logs
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-sse-performance-logs
path: test/s3/sse/weed-test*.log
@@ -389,7 +389,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-volume-encryption-logs
path: /tmp/seaweedfs-sse-*.log
+7 -68
View File
@@ -66,7 +66,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: s3-tables-test-logs
path: test/s3tables/table-buckets/test-output.log
@@ -122,7 +122,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: iceberg-catalog-test-logs
path: test/s3tables/catalog/test-output.log
@@ -188,73 +188,12 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: trino-iceberg-catalog-test-logs
path: test/s3tables/catalog_trino/test-output.log
retention-days: 3
polaris-integration-tests:
name: Polaris Integration Tests
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version-file: 'go.mod'
id: go
- name: Run go mod tidy
run: go mod tidy
- name: Install SeaweedFS
run: |
go install -buildvcs=false ./weed
- name: Pre-pull Polaris image
run: docker pull apache/polaris:latest
- name: Run Polaris Integration Tests
timeout-minutes: 25
run: |
set -x
set -o pipefail
echo "=== System Information ==="
uname -a
free -h
df -h
echo "=== Starting Polaris Tests ==="
go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || {
echo "Polaris integration tests failed"
exit 1
}
- name: Show test output on failure
if: failure()
working-directory: test/s3tables/polaris
run: |
echo "=== Test Output ==="
if [ -f test-output.log ]; then
tail -200 test-output.log
fi
echo "=== Process information ==="
ps aux | grep -E "(weed|test|docker)" || true
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: polaris-test-logs
path: test/s3tables/polaris/test-output.log
retention-days: 3
spark-iceberg-catalog-tests:
name: Spark Iceberg Catalog Integration Tests
runs-on: ubuntu-22.04
@@ -315,7 +254,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: spark-iceberg-catalog-test-logs
path: test/s3tables/catalog_spark/test-output.log
@@ -383,7 +322,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: risingwave-catalog-test-logs
path: test/s3tables/catalog_risingwave/test-output.log
@@ -449,7 +388,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: sts-integration-test-logs
path: test/s3tables/sts_integration/test-output.log
@@ -518,7 +457,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: lakekeeper-integration-test-logs
path: test/s3tables/lakekeeper/test-output.log
@@ -125,7 +125,7 @@ jobs:
- name: Upload test results
if: always()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: spark-test-results
path: test/java/spark/target/surefire-reports/
@@ -103,7 +103,7 @@ jobs:
- name: Upload server logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: seaweedfs-logs
# Note: actions don't use defaults.run.working-directory, so path is relative to workspace root
+1 -1
View File
@@ -106,7 +106,7 @@ jobs:
- name: Upload test logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: tus-test-logs
path: |
@@ -90,7 +90,7 @@ jobs:
- name: Archive logs on failure
if: failure()
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: volume-server-integration-test-logs
path: /tmp/volume-server-it-logs/
+28 -28
View File
@@ -26,7 +26,7 @@ require (
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-redsync/redsync/v4 v4.16.0
github.com/go-redsync/redsync/v4 v4.15.0
github.com/go-sql-driver/mysql v1.9.3
github.com/go-zookeeper/zk v1.0.3 // indirect
github.com/golang/protobuf v1.5.4
@@ -89,20 +89,20 @@ require (
go.etcd.io/etcd/client/v3 v3.6.7
go.mongodb.org/mongo-driver v1.17.6
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.45.0
gocloud.dev/pubsub/natspubsub v0.45.0
gocloud.dev v0.44.0
gocloud.dev/pubsub/natspubsub v0.44.0
gocloud.dev/pubsub/rabbitpubsub v0.44.0
golang.org/x/crypto v0.48.0
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
golang.org/x/image v0.36.0
golang.org/x/net v0.49.0
golang.org/x/oauth2 v0.35.0
golang.org/x/oauth2 v0.34.0
golang.org/x/sys v0.41.0
golang.org/x/text v0.34.0 // indirect
golang.org/x/tools v0.41.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
google.golang.org/api v0.258.0
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 // indirect
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect
google.golang.org/grpc v1.78.0
google.golang.org/protobuf v1.36.11
gopkg.in/inf.v0 v0.9.1 // indirect
@@ -138,7 +138,8 @@ require (
github.com/hashicorp/raft-boltdb/v2 v2.3.1
github.com/hashicorp/vault/api v1.22.0
github.com/jhump/protoreflect v1.18.0
github.com/linkedin/goavro/v2 v2.15.0
github.com/lib/pq v1.11.1
github.com/linkedin/goavro/v2 v2.14.1
github.com/mattn/go-sqlite3 v1.14.34
github.com/minio/crc64nvme v1.1.1
github.com/orcaman/concurrent-map/v2 v2.0.1
@@ -150,7 +151,7 @@ require (
github.com/redis/go-redis/v9 v9.18.0
github.com/schollz/progressbar/v3 v3.19.0
github.com/seaweedfs/go-fuse/v2 v2.9.1
github.com/shirou/gopsutil/v4 v4.26.2
github.com/shirou/gopsutil/v4 v4.26.1
github.com/tarantool/go-tarantool/v2 v2.4.1
github.com/testcontainers/testcontainers-go v0.39.0
github.com/tikv/client-go/v2 v2.0.7
@@ -171,7 +172,7 @@ require (
atomicgo.dev/keyboard v0.2.9 // indirect
atomicgo.dev/schedule v0.1.0 // indirect
cloud.google.com/go/longrunning v0.7.0 // indirect
cloud.google.com/go/pubsub/v2 v2.3.0 // indirect
cloud.google.com/go/pubsub/v2 v2.2.1 // indirect
dario.cat/mergo v1.0.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
@@ -236,7 +237,6 @@ require (
github.com/klauspost/asmfmt v1.3.2 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.11.1 // indirect
github.com/lithammer/fuzzysearch v1.1.8 // indirect
github.com/lithammer/shortuuid/v3 v3.0.7 // indirect
github.com/magiconair/properties v1.8.10 // indirect
@@ -279,10 +279,10 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 // indirect
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
go.opentelemetry.io/proto/otlp v1.7.0 // indirect
go.uber.org/mock v0.5.2 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
@@ -292,12 +292,12 @@ require (
)
require (
cel.dev/expr v0.25.1 // indirect
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go/auth v0.17.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
cloud.google.com/go/iam v1.5.3 // indirect
cloud.google.com/go/monitoring v1.24.3 // indirect
cloud.google.com/go/monitoring v1.24.2 // indirect
filippo.io/edwards25519 v1.1.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
@@ -327,7 +327,7 @@ require (
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
@@ -336,11 +336,11 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
github.com/aws/smithy-go v1.24.0
github.com/boltdb/bolt v1.3.1 // indirect
github.com/bradenaw/juniper v0.15.3 // indirect
@@ -352,7 +352,7 @@ require (
github.com/cloudinary/cloudinary-go/v2 v2.13.0 // indirect
github.com/cloudsoda/go-smb2 v0.0.0-20250228001242-d4c70e6251cc // indirect
github.com/cloudsoda/sddl v0.0.0-20250224235906-926454e91efc // indirect
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e // indirect
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f // indirect
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
github.com/creasty/defaults v1.8.0 // indirect
github.com/cronokirby/saferith v0.33.0 // indirect
@@ -360,11 +360,11 @@ require (
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
github.com/ebitengine/purego v0.10.0 // indirect
github.com/ebitengine/purego v0.9.1 // indirect
github.com/elastic/gosigar v0.14.3 // indirect
github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/fatih/color v1.18.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
@@ -428,8 +428,8 @@ require (
github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/nats.go v1.48.0 // indirect
github.com/nats-io/nkeys v0.4.12 // indirect
github.com/nats-io/nats.go v1.43.0 // indirect
github.com/nats-io/nkeys v0.4.11 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/ncw/swift/v2 v2.0.5 // indirect
@@ -493,11 +493,11 @@ require (
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect
go.opentelemetry.io/otel v1.40.0 // indirect
go.opentelemetry.io/otel/metric v1.40.0 // indirect
go.opentelemetry.io/otel/sdk v1.40.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.40.0 // indirect
go.opentelemetry.io/otel/trace v1.40.0 // indirect
go.opentelemetry.io/otel v1.38.0 // indirect
go.opentelemetry.io/otel/metric v1.38.0 // indirect
go.opentelemetry.io/otel/sdk v1.38.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.38.0 // indirect
go.opentelemetry.io/otel/trace v1.38.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect
golang.org/x/term v0.40.0 // indirect
+68 -72
View File
@@ -6,8 +6,8 @@ atomicgo.dev/keyboard v0.2.9 h1:tOsIid3nlPLZ3lwgG8KZMp/SFmr7P0ssEN5JUsm78K8=
atomicgo.dev/keyboard v0.2.9/go.mod h1:BC4w9g00XkxH/f1HXhW2sXmJFOCWbKn9xrOunSFtExQ=
atomicgo.dev/schedule v0.1.0 h1:nTthAbhZS5YZmgYbb2+DH8uQIZcTlIrd4eYr3UQxEjs=
atomicgo.dev/schedule v0.1.0/go.mod h1:xeUa3oAkiuHYh8bKiQBRojqAMq3PXXbJujjb0hw8pEU=
cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY=
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -310,8 +310,8 @@ cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6
cloud.google.com/go/lifesciences v0.8.0/go.mod h1:lFxiEOMqII6XggGbOnKiyZ7IBwoIqA84ClvoezaA/bo=
cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw=
cloud.google.com/go/logging v1.7.0/go.mod h1:3xjP2CjkM3ZkO73aj4ASA5wRPGGCRrPIAeNqVNkzY8M=
cloud.google.com/go/logging v1.13.1 h1:O7LvmO0kGLaHY/gq8cV7T0dyp6zJhYAOtZPX4TF3QtY=
cloud.google.com/go/logging v1.13.1/go.mod h1:XAQkfkMBxQRjQek96WLPNze7vsOmay9H5PqfsNYDqvw=
cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc=
cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA=
cloud.google.com/go/longrunning v0.1.1/go.mod h1:UUFxuDWkv22EuY93jjmDMFT5GPQKeFVJBIF6QlTqdsE=
cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc=
cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
@@ -338,8 +338,8 @@ cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJP
cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk=
cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4=
cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w=
cloud.google.com/go/monitoring v1.24.3 h1:dde+gMNc0UhPZD1Azu6at2e79bfdztVDS5lvhOdsgaE=
cloud.google.com/go/monitoring v1.24.3/go.mod h1:nYP6W0tm3N9H/bOw8am7t62YTzZY+zUeQ+Bi6+2eonI=
cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM=
cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U=
cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA=
cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o=
cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM=
@@ -393,8 +393,8 @@ cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhz
cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
cloud.google.com/go/pubsub v1.50.1 h1:fzbXpPyJnSGvWXF1jabhQeXyxdbCIkXTpjXHy7xviBM=
cloud.google.com/go/pubsub v1.50.1/go.mod h1:6YVJv3MzWJUVdvQXG081sFvS0dWQOdnV+oTo++q/xFk=
cloud.google.com/go/pubsub/v2 v2.3.0 h1:DgAN907x+sP0nScYfBzneRiIhWoXcpCD8ZAut8WX9vs=
cloud.google.com/go/pubsub/v2 v2.3.0/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
cloud.google.com/go/pubsub/v2 v2.2.1 h1:3brZcshL3fIiD1qOxAE2QW9wxsfjioy014x4yC9XuYI=
cloud.google.com/go/pubsub/v2 v2.2.1/go.mod h1:O5f0KHG9zDheZAd3z5rlCRhxt2JQtB+t/IYLKK3Bpvw=
cloud.google.com/go/pubsublite v1.5.0/go.mod h1:xapqNQ1CuLfGi23Yda/9l4bBCKz/wC3KIJ5gKcxveZg=
cloud.google.com/go/pubsublite v1.6.0/go.mod h1:1eFCS0U11xlOuMFV/0iBqw3zP12kddMeCbj/F3FSj9k=
cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4=
@@ -504,8 +504,8 @@ cloud.google.com/go/tpu v1.5.0/go.mod h1:8zVo1rYDFuW2l4yZVY0R0fb/v44xLh3llq7RuV6
cloud.google.com/go/trace v1.3.0/go.mod h1:FFUE83d9Ca57C+K8rDl/Ih8LwOzWIV1krKgxg6N0G28=
cloud.google.com/go/trace v1.4.0/go.mod h1:UG0v8UBqzusp+z63o7FK74SdFE+AXpCLdFb1rshXG+Y=
cloud.google.com/go/trace v1.8.0/go.mod h1:zH7vcsbAhklH8hWFig58HvxcxyQbaIqMarMg9hn5ECA=
cloud.google.com/go/trace v1.11.7 h1:kDNDX8JkaAG3R2nq1lIdkb7FCSi1rCmsEtKVsty7p+U=
cloud.google.com/go/trace v1.11.7/go.mod h1:TNn9d5V3fQVf6s4SCveVMIBS2LJUqo73GACmq/Tky0s=
cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4=
cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI=
cloud.google.com/go/translate v1.3.0/go.mod h1:gzMUwRjvOqj5i69y/LYLd8RrNQk+hOmIXTi9+nb3Djs=
cloud.google.com/go/translate v1.4.0/go.mod h1:06Dn/ppvLD6WvA5Rhdp029IX2Mi3Mn7fpMRLPvXT5Wg=
cloud.google.com/go/translate v1.6.0/go.mod h1:lMGRudH1pu7I3n3PETiOB2507gf3HnfLV8qlkHZEyos=
@@ -687,8 +687,6 @@ github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUS
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op h1:Ucf+QxEKMbPogRO5guBNe5cgd9uZgfoJLOYs8WWhtjM=
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4=
@@ -726,8 +724,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUT
github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12 h1:Zy6Tme1AA13kX8x3CnkHx5cqdGWGaj/anwOiWGnA0Xo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.12/go.mod h1:ql4uXYKoTM9WUAUSmthY4AtPVrlTBZOvnBJTiCUdPxI=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4 h1:2fjfz3/G9BRvIKuNZ655GwzpklC2kEH0cowZQGO7uBg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.4/go.mod h1:Ymws824lvMypLFPwyyUXM52SXuGgxpu0+DISLfKvB+c=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
@@ -748,10 +746,10 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0 h1:MIWra+MSq53CFaXXAywB2qg9YvVZi
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7 h1:fovS7qGMT+BBSuifkySdVaMWxXTyaYT6qaBx/1y6Ij4=
github.com/aws/aws-sdk-go-v2/service/sns v1.39.7/go.mod h1:gFahrattA8ulEtiS4XL/fQiQ77l+Urc52Y96/r1e6ks=
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17 h1:ZNMxVFPayuHe14u/vn+BwLi3wxQvxcNTw8WdPv2gqBc=
github.com/aws/aws-sdk-go-v2/service/sqs v1.42.17/go.mod h1:ZxqweFQ2w6NNznWMUvWV9AvkAfM6J8F/MC250Mb4n1I=
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7 h1:OBuZE9Wt8h2imuRktu+WfjiTGrnYdCIJg8IX92aalHE=
github.com/aws/aws-sdk-go-v2/service/sns v1.34.7/go.mod h1:4WYoZAhHt+dWYpoOQUgkUKfuQbE6Gg/hW4oXE0pKS9U=
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8 h1:80dpSqWMwx2dAm30Ib7J6ucz1ZHfiv5OCRwN/EnCOXQ=
github.com/aws/aws-sdk-go-v2/service/sqs v1.38.8/go.mod h1:IzNt/udsXlETCdvBOL0nmyMe2t9cGmXmZgsdoZGYYhI=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
@@ -857,8 +855,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20230105202645-06c439db220b/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20230310173818-32f1caf87195/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e h1:gt7U1Igw0xbJdyaCM5H2CnlAlPSkzrhsebQB6WQWjLA=
github.com/cncf/xds/go v0.0.0-20251110193048-8bfbf64dc13e/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg=
github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc=
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
@@ -974,8 +972,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203 h1:XBBHcIb256gUJtLmY22n99HaZTz+r2Z51xUPi01m3wg=
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203/go.mod h1:E1jcSv8FaEny+OP/5k9UxZVw9YFWGj7eI4KR/iOBqCg=
github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo=
@@ -1002,8 +1000,8 @@ github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJ
github.com/envoyproxy/go-control-plane v0.11.0/go.mod h1:VnHyVMpzcLvCFt9yUz1UnCwHLhwx1WguiVDV7pTG/tI=
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g=
github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98=
github.com/envoyproxy/go-control-plane/envoy v1.35.0 h1:ixjkELDE+ru6idPxcHLj8LBVc2bFP7iBytj353BoHUo=
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
@@ -1126,8 +1124,8 @@ github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOr
github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-redsync/redsync/v4 v4.16.0 h1:bNcOzeHH9d3s6pghU9NJFMPrQa41f5Nx3L4YKr3BdEU=
github.com/go-redsync/redsync/v4 v4.16.0/go.mod h1:V4gagqgyASWBZuwx4xGzu72aZNb/6Mo05byUa3mVmKQ=
github.com/go-redsync/redsync/v4 v4.15.0 h1:KH/XymuxSV7vyKs6z1Cxxj+N+N18JlPxgXeP6x4JY54=
github.com/go-redsync/redsync/v4 v4.15.0/go.mod h1:qNp+lLs3vkfZbtA/aM/OjlZHfEr5YTAYhRktFPKHC7s=
github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM=
github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
@@ -1239,8 +1237,6 @@ github.com/google/go-replayers/grpcreplay v1.3.0 h1:1Keyy0m1sIpqstQmgz307zhiJ1pV
github.com/google/go-replayers/grpcreplay v1.3.0/go.mod h1:v6NgKtkijC0d3e3RW8il6Sy5sqRVUwoQa4mHOGEy8DI=
github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQEEioJvFYxYcLRKzk=
github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -1518,8 +1514,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
github.com/linkedin/goavro/v2 v2.15.0 h1:pDj1UrjUOO62iXhgBiE7jQkpNIc5/tA5eZsgolMjgVI=
github.com/linkedin/goavro/v2 v2.15.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
github.com/linkedin/goavro/v2 v2.14.1 h1:/8VjDpd38PRsy02JS0jflAu7JZPfJcGTwqWgMkFS2iI=
github.com/linkedin/goavro/v2 v2.14.1/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
github.com/linxGnu/grocksdb v1.10.7 h1:fCi4qvZWo04VgFwGWmO8HQJgUVounJBy+C2TMVPU/ho=
github.com/linxGnu/grocksdb v1.10.7/go.mod h1:OLQKZwiKwaJiAVCsOzWKvwiLwfZ5Vz8Md5TYR7t7pM8=
github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=
@@ -1567,8 +1563,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8D
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -1626,14 +1622,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
github.com/nats-io/nats-server/v2 v2.11.12 h1:jGDXTkcjqQ5fCRstwIxvv1K0RHfftFUoSCT/iIZcqOc=
github.com/nats-io/nats-server/v2 v2.11.12/go.mod h1:5MCp/pqm5SEfsvVZ31ll1088ZTwEUdvRX1Hmh/mTTDg=
github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U=
github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
github.com/nats-io/jwt/v2 v2.5.0 h1:WQQ40AAlqqfx+f6ku+i0pOVm+ASirD4fUh+oQsiE9Ak=
github.com/nats-io/jwt/v2 v2.5.0/go.mod h1:24BeQtRwxRV8ruvC4CojXlx/WQ/VjuwlYiH+vu/+ibI=
github.com/nats-io/nats-server/v2 v2.9.23 h1:6Wj6H6QpP9FMlpCyWUaNu2yeZ/qGj+mdRkZ1wbikExU=
github.com/nats-io/nats-server/v2 v2.9.23/go.mod h1:wEjrEy9vnqIGE4Pqz4/c75v9Pmaq7My2IgFmnykc4C0=
github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug=
github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
@@ -1804,10 +1800,10 @@ github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6ha
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
github.com/redis/rueidis v1.0.71 h1:pODtnAR5GAB7j4ekhldZ29HKOxe4Hph0GTDGk1ayEQY=
github.com/redis/rueidis v1.0.71/go.mod h1:lfdcZzJ1oKGKL37vh9fO3ymwt+0TdjkkUCJxbgpmcgQ=
github.com/redis/rueidis/rueidiscompat v1.0.71 h1:wNZ//kEjMZgBM0KCk7ncOX8KmAgROU2kDdDNpwheG4w=
github.com/redis/rueidis/rueidiscompat v1.0.71/go.mod h1:esmCLJvaRzZoKlgB82G1bY7Iky5TnO9Rz+NlhbEccFI=
github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
github.com/redis/rueidis/rueidiscompat v1.0.69/go.mod h1:iC4Y8DoN0Uth0Uezg9e2trvNRC7QAgGeuP2OPLb5ccI=
github.com/rekby/fixenv v0.3.2/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM=
github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c=
@@ -1861,8 +1857,8 @@ github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQ
github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc=
github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI=
github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE=
github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI=
github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
github.com/shirou/gopsutil/v4 v4.26.1 h1:TOkEyriIXk2HX9d4isZJtbjXbEjf5qyKPAzbzY0JWSo=
github.com/shirou/gopsutil/v4 v4.26.1/go.mod h1:medLI9/UNAb0dOI9Q3/7yWSqKkj00u+1tgY8nvv41pc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
@@ -2134,35 +2130,35 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.56.0/go.mod h1:3qi2EEwMgB4xnKgPLqsDP3j9qxnHDZeHsnAxfjQqTko=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg=
go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0 h1:vl9obrcoWVKp/lwl8tRE33853I8Xru9HFbw/skNeLs8=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0/go.mod h1:GAXRxmLJcVM3u22IjTg74zWBrRCKq8BnOqUVLodpcpw=
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0 h1:zG8GlgXCJQd5BU98C0hZnBbElszTmUgCNCfYneaDL0A=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.37.0/go.mod h1:hOfBCz8kv/wuq73Mx2H2QnWokh/kHZxkh6SNF2bdKtw=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 h1:ZsXq73BERAiNuuFXYqP4MR5hBrjXfMGSO+Cx7qoOZiM=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0/go.mod h1:hg1zaDMpyZJuUzjFxFsRYBoccE86tM9Uf4IqNMUxvrY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4Dc5leUqENgGuQImwLo4WnuXFPetmPpkLi2IrX54=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 h1:Ahq7pZmv87yiyn3jeFz/LekZmPLLdKejuO3NcK9MssM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0/go.mod h1:MJTqhM0im3mRLw1i8uGHnCvUEeS7VwRyxlLC78PA18M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 h1:EtFWSnwW9hGObjkIdmlnWSydO+Qs8OwzfzXLUPg4xOc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0/go.mod h1:QjUEoiGCPkvFZ/MjK6ZZfNOS6mfVEVKYE99dFhuN2LI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0 h1:lUsI2TYsQw2r1IASwoROaCnjdj2cvC2+Jbxvk6nHnWU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.31.0/go.mod h1:2HpZxxQurfGxJlJDblybejHB6RX6pmExPNe517hREw4=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0 h1:wm/Q0GAAykXv83wzcKzGGqAnnfLFyFe7RslekZuv+VI=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.38.0/go.mod h1:ra3Pa40+oKjvYh+ZD3EdxFZZB0xdMfuileHAm4nNN7w=
go.opentelemetry.io/otel/exporters/zipkin v1.36.0 h1:s0n95ya5tOG03exJ5JySOdJFtwGo4ZQ+KeY7Zro4CLI=
go.opentelemetry.io/otel/exporters/zipkin v1.36.0/go.mod h1:m9wRxtKA2MZ1HcnNC4BKI+9aYe434qRZTCvI7QGUN7Y=
go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw=
go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg=
go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@@ -2186,10 +2182,10 @@ go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
gocloud.dev v0.45.0 h1:WknIK8IbRdmynDvara3Q7G6wQhmEiOGwpgJufbM39sY=
gocloud.dev v0.45.0/go.mod h1:0kXKmkCLG6d31N7NyLZWzt7jDSQura9zD/mWgiB6THI=
gocloud.dev/pubsub/natspubsub v0.45.0 h1:kfCupVejeynIQcS1GaIvkY3Nj/acLlM5yPWyxZN8wJ8=
gocloud.dev/pubsub/natspubsub v0.45.0/go.mod h1:WU5SMDWuF7CCr2U8UpbrEONYYFOmvigCAvTpOOpUvYE=
gocloud.dev v0.44.0 h1:iVyMAqFl2r6xUy7M4mfqwlN+21UpJoEtgHEcfiLMUXs=
gocloud.dev v0.44.0/go.mod h1:ZmjROXGdC/eKZLF1N+RujDlFRx3D+4Av2thREKDMVxY=
gocloud.dev/pubsub/natspubsub v0.44.0 h1:1Us76ckkdgtiE1p1rJZ+38b9TQP051bmjAiQlFQzYrM=
gocloud.dev/pubsub/natspubsub v0.44.0/go.mod h1:PvVAGIhL14PWGwWIXX/zAK42ixr2/PKP4Q4yMiAUraQ=
gocloud.dev/pubsub/rabbitpubsub v0.44.0 h1:MpRIO6XJ/JTqrlUWt3CxwDe1LvaiXUVu4sS5cv4f/AM=
gocloud.dev/pubsub/rabbitpubsub v0.44.0/go.mod h1:BB9+qT3r6g4M5+4asiXaEeqw4QAOzsWusO5krYaqkdA=
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
@@ -2380,8 +2376,8 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -2844,8 +2840,8 @@ google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc/go.mod h1:RGgjbofJ
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44/go.mod h1:8B0gmkoRebU8ukX6HP+4wrVQUY1+6PkQ44BSyIlflHA=
google.golang.org/genproto v0.0.0-20230222225845-10f96fb3dbec/go.mod h1:3Dl5ZL0q0isWJt+FVcfpQyirqemEuLAK/iFvg1UP1Hw=
google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s=
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846 h1:dDbsTLIK7EzwUq36kCSAsk0slouq/S0tWHeeGi97cD8=
google.golang.org/genproto v0.0.0-20251124214823-79d6a2a48846/go.mod h1:PP0g88Dz3C7hRAfbQCQggeWAXjuqGsNPLE4s7jh0RGU=
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 h1:LvZVVaPE0JSqL+ZWb6ErZfnEOKIqqFWUJE2D0fObSmc=
google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9/go.mod h1:QFOrLhdAe2PsTp3vQY4quuLKTi9j3XG3r6JPPaw7MSc=
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846 h1:ZdyUkS9po3H7G0tuh955QVyyotWvOD4W0aEapeGeUYk=
google.golang.org/genproto/googleapis/api v0.0.0-20251124214823-79d6a2a48846/go.mod h1:Fk4kyraUvqD7i5H6S43sj2W98fbZa75lpZz/eUyhfO0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20251213004720-97cd9d5aeac2 h1:2I6GHUeJ/4shcDpoUlLs/2WPnhg7yJwvXtqcMJt9liA=
@@ -100,19 +100,12 @@ filer:
# S3 gateway (if enabled)
s3:
enabled: true
replicas: 1
port: 8333
enableAuth: true
podSecurityContext:
enabled: true
# On OpenShift, we omit runAsUser/runAsGroup/fsGroup to let the admission
# controller assign them automatically based on the namespace's SCC.
runAsNonRoot: true
logs:
type: "emptyDir"
containerSecurityContext:
enabled: true
allowPrivilegeEscalation: false
@@ -118,10 +118,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.admin "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.admin.extraEnvironmentVars }}
{{- range $key, $value := .Values.admin.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -130,6 +128,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -96,10 +96,8 @@ spec:
- name: WEED_GRPC_CA
value: /usr/local/share/ca-certificates/client/ca.crt
{{- end }}
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.cosi "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.cosi.extraEnvironmentVars }}
{{- range $key, $value := .Values.cosi.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -108,6 +106,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
volumeMounts:
- mountPath: /var/lib/cosi
name: socket
@@ -114,10 +114,8 @@ spec:
optional: true
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.filer "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.filer.extraEnvironmentVars }}
{{- range $key, $value := .Values.filer.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -126,6 +124,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.filer.secretExtraEnvironmentVars }}
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
- name: {{ $key }}
@@ -69,7 +69,9 @@ spec:
priorityClassName: {{ .Values.master.priorityClassName | quote }}
{{- end }}
enableServiceLinks: false
{{- if .Values.global.createClusterRole }}
serviceAccountName: {{ .Values.master.serviceAccountName | default (include "seaweedfs.serviceAccountName" .) | quote }} # for deleting statefulset pods after migration
{{- end }}
{{- if .Values.master.initContainers }}
initContainers:
{{ tpl .Values.master.initContainers . | nindent 8 | trim }}
@@ -96,10 +98,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.master "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.master.extraEnvironmentVars }}
{{- range $key, $value := .Values.master.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -108,6 +108,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -90,10 +90,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.s3 "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.s3.extraEnvironmentVars }}
{{- range $key, $value := .Values.s3.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -102,6 +100,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -90,10 +90,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.sftp "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.sftp.extraEnvironmentVars }}
{{- range $key, $value := .Values.sftp.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -102,6 +100,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -59,18 +59,6 @@ Inject extra environment vars in the format key:value, if populated
{{- end -}}
{{- end -}}
{{- define "seaweedfs.mergeExtraEnvironmentVars" -}}
{{- $global := ((.global | default dict).extraEnvironmentVars | default dict) -}}
{{- $component := ((.component | default dict).extraEnvironmentVars | default dict) -}}
{{- $target := .target -}}
{{- range $key, $value := $global }}
{{- $_ := set $target $key $value }}
{{- end }}
{{- range $key, $value := $component }}
{{- $_ := set $target $key $value }}
{{- end }}
{{- end -}}
{{/* Return the proper filer image */}}
{{- define "filer.image" -}}
{{- if .Values.filer.imageOverride -}}
@@ -69,7 +69,9 @@ spec:
priorityClassName: {{ $volume.priorityClassName | quote }}
{{- end }}
enableServiceLinks: false
{{- if $.Values.global.createClusterRole }}
serviceAccountName: {{ $volume.serviceAccountName | default (include "seaweedfs.serviceAccountName" $) | quote }} # for deleting statefulset pods after migration
{{- end }}
{{- $initContainers_exists := include "volume.initContainers_exists" $ -}}
{{- if $initContainers_exists }}
initContainers:
@@ -116,10 +118,8 @@ spec:
fieldPath: status.hostIP
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" $ }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" $.Values.global "component" $volume "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if $volume.extraEnvironmentVars }}
{{- range $key, $value := $volume.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -128,6 +128,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if $.Values.global.extraEnvironmentVars }}
{{- range $key, $value := $.Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -93,10 +93,8 @@ spec:
fieldPath: metadata.namespace
- name: SEAWEEDFS_FULLNAME
value: "{{ include "seaweedfs.fullname" . }}"
{{- $mergedExtraEnvironmentVars := dict }}
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global "component" .Values.worker "target" $mergedExtraEnvironmentVars) }}
{{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }}
{{- $value := index $mergedExtraEnvironmentVars $key }}
{{- if .Values.worker.extraEnvironmentVars }}
{{- range $key, $value := .Values.worker.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
@@ -105,6 +103,18 @@ spec:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
{{- if .Values.global.extraEnvironmentVars }}
{{- range $key, $value := .Values.global.extraEnvironmentVars }}
- name: {{ $key }}
{{- if kindIs "string" $value }}
value: {{ tpl $value $ | quote }}
{{- else }}
valueFrom:
{{ toYaml $value | nindent 16 | trim }}
{{- end -}}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"
@@ -135,7 +135,7 @@ func ensureEnvironment(t *testing.T) {
// 6. Start Admin
os.RemoveAll(filepath.Join("tmp", "admin"))
os.MkdirAll(filepath.Join("tmp", "admin"), 0755)
startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin", "-scheduler.idleSleep=2")
startWeed(t, "admin", "admin", "-master=localhost:9333", "-port=23646", "-dataDir=./tmp/admin")
waitForUrl(t, AdminUrl+"/health", 60)
t.Log("Environment started successfully")
+1 -1
View File
@@ -21,7 +21,7 @@
<scala.binary.version>2.12</scala.binary.version>
<junit.version>4.13.2</junit.version>
<seaweedfs.hadoop3.client.version>4.00</seaweedfs.hadoop3.client.version>
<jackson.version>2.18.6</jackson.version> <!-- Upgraded from 2.15.3 -->
<jackson.version>2.18.2</jackson.version> <!-- Upgraded from 2.15.3 -->
<netty.version>4.1.129.Final</netty.version> <!-- Upgraded to 4.1.125.Final for security fixes (CVE in netty-codec < 4.1.125.Final, netty-codec-http2 <= 4.1.123.Final) -->
<parquet.version>1.15.2</parquet.version> <!-- Upgraded to 1.15.2 for security fix -->
<parquet.format.version>2.12.0</parquet.format.version>
+118 -466
View File
@@ -1,32 +1,15 @@
package lakekeeper
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
)
type TestEnvironment struct {
@@ -48,19 +31,15 @@ type TestEnvironment struct {
secretKey string
}
const (
lakekeeperRegion = "us-east-1"
lakekeeperRoleArn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
lakekeeperSessionName = "lakekeeper-session"
)
type lakekeeperSession struct {
endpoint string
region string
creds aws.Credentials
}
func TestLakekeeperIntegration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
if !testutil.HasDocker() {
t.Skip("Docker not available, skipping Lakekeeper integration test")
}
env := NewTestEnvironment(t)
defer env.Cleanup(t)
@@ -72,18 +51,6 @@ func TestLakekeeperIntegration(t *testing.T) {
runLakekeeperRepro(t, env)
}
func TestLakekeeperTableBucketIntegration(t *testing.T) {
env := NewTestEnvironment(t)
defer env.Cleanup(t)
fmt.Printf(">>> Starting SeaweedFS with Lakekeeper configuration...\n")
env.StartSeaweedFS(t)
fmt.Printf(">>> SeaweedFS started.\n")
// Run python script in docker to test STS and S3 Tables operations
runLakekeeperTableBucketRepro(t, env)
}
func NewTestEnvironment(t *testing.T) *TestEnvironment {
t.Helper()
@@ -251,439 +218,124 @@ func (env *TestEnvironment) Cleanup(t *testing.T) {
func runLakekeeperRepro(t *testing.T, env *TestEnvironment) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
scriptContent := fmt.Sprintf(`
import boto3
import botocore.config
import botocore
from botocore.exceptions import ClientError
import os
import sys
import time
import logging
session, err := newLakekeeperSession(ctx, env)
if err != nil {
t.Fatalf("AssumeRole failed: %v", err)
# Enable botocore debug logging to see signature calculation
logging.basicConfig(level=logging.DEBUG)
botocore.session.get_session().set_debug_logger()
print("Starting Lakekeeper repro test...")
endpoint_url = "http://host.docker.internal:%d"
access_key = "%s"
secret_key = "%s"
region = "us-east-1"
print(f"Connecting to {endpoint_url}")
try:
config = botocore.config.Config(
retries={'max_attempts': 3}
)
sts = boto3.client(
'sts',
endpoint_url=endpoint_url,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region,
config=config
)
role_arn = "arn:aws:iam::000000000000:role/LakekeeperVendedRole"
session_name = "lakekeeper-session"
print(f"Calling AssumeRole on {role_arn} with POST body...")
# Standard boto3 call sends parameters in POST body
response = sts.assume_role(
RoleArn=role_arn,
RoleSessionName=session_name
)
creds = response['Credentials']
access_key_id = creds['AccessKeyId']
secret_access_key = creds['SecretAccessKey']
session_token = creds['SessionToken']
print(f"Success! Got credentials with prefix: {access_key_id[:4]}")
if not access_key_id.startswith("ASIA"):
print(f"FAILED: Expected ASIA prefix, got {access_key_id}")
sys.exit(1)
print("Verifying S3 operations with vended credentials...")
s3 = boto3.client(
's3',
endpoint_url=endpoint_url,
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key,
aws_session_token=session_token,
region_name=region,
config=config
)
bucket = "lakekeeper-vended-bucket"
print(f"Creating bucket {bucket}...")
s3.create_bucket(Bucket=bucket)
print("Listing buckets...")
response = s3.list_buckets()
buckets = [b['Name'] for b in response['Buckets']]
print(f"Found buckets: {buckets}")
if bucket not in buckets:
print(f"FAILED: Bucket {bucket} not found in list")
sys.exit(1)
print("SUCCESS: Lakekeeper flow verified!")
sys.exit(0)
except Exception as e:
print(f"FAILED: {e}")
# Print more details if it is a ClientError
if hasattr(e, 'response'):
print(f"Response: {e.response}")
sys.exit(1)
`, env.s3Port, env.accessKey, env.secretKey)
scriptPath := filepath.Join(env.dataDir, "lakekeeper_repro.py")
if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
t.Fatalf("Failed to write python script: %v", err)
}
s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
if err != nil {
t.Fatalf("Create S3 client failed: %v", err)
}
containerName := "seaweed-lakekeeper-client-" + fmt.Sprintf("%d", time.Now().UnixNano())
bucketName := fmt.Sprintf("lakekeeper-vended-bucket-%d", time.Now().UnixNano())
if _, err := s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
}); err != nil {
t.Fatalf("CreateBucket failed: %v", err)
}
// Create a context with timeout for the docker run command
dockerCtx, dockerCancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer dockerCancel()
bucketCreated := true
defer func() {
if !bucketCreated {
return
}
_, _ = s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
Bucket: aws.String(bucketName),
})
}()
listResp, err := s3Client.ListBuckets(ctx, &s3.ListBucketsInput{})
if err != nil {
t.Fatalf("ListBuckets failed: %v", err)
}
found := false
for _, bucket := range listResp.Buckets {
if aws.ToString(bucket.Name) == bucketName {
found = true
break
}
}
if !found {
t.Fatalf("Bucket %s not found in list", bucketName)
}
if _, err := s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
Bucket: aws.String(bucketName),
}); err != nil {
t.Fatalf("DeleteBucket failed: %v", err)
}
bucketCreated = false
}
func runLakekeeperTableBucketRepro(t *testing.T, env *TestEnvironment) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
session, err := newLakekeeperSession(ctx, env)
if err != nil {
t.Fatalf("AssumeRole failed: %v", err)
}
client := newS3TablesClient(session.endpoint, session.region, session.creds)
bucketName := fmt.Sprintf("lakekeeper-table-bucket-%d", time.Now().UnixNano())
bucketARN, err := client.CreateTableBucket(ctx, bucketName)
if err != nil {
t.Fatalf("CreateTableBucket failed: %v", err)
}
bucketDeleted := false
namespaceCreated := false
tableCreated := false
namespaceName := fmt.Sprintf("lakekeeper_ns_%d", time.Now().UnixNano())
tableName := fmt.Sprintf("lakekeeper_table_%d", time.Now().UnixNano())
defer func() {
if tableCreated {
if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
t.Logf("Failed to delete table: %v", err)
}
}
if namespaceCreated {
if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
t.Logf("Failed to delete namespace: %v", err)
}
}
if bucketDeleted {
return
}
if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
t.Logf("Failed to delete table bucket: %v", err)
}
}()
buckets, err := client.ListTableBuckets(ctx)
if err != nil {
t.Fatalf("ListTableBuckets failed: %v", err)
}
found := false
for _, b := range buckets {
if b.Name == bucketName {
found = true
break
}
}
if !found {
t.Fatalf("Created table bucket %s not found in list", bucketName)
}
if _, err := client.GetTableBucket(ctx, bucketARN); err != nil {
t.Fatalf("GetTableBucket failed: %v", err)
}
if err := client.CreateNamespace(ctx, bucketARN, namespaceName); err != nil {
t.Fatalf("CreateNamespace failed: %v", err)
}
namespaceCreated = true
if err := client.CreateTable(ctx, bucketARN, namespaceName, tableName); err != nil {
t.Fatalf("CreateTable failed: %v", err)
}
tableCreated = true
s3Client, err := newS3Client(ctx, session.endpoint, session.region, session.creds)
if err != nil {
t.Fatalf("Create S3 client failed: %v", err)
}
objectKey := fmt.Sprintf("%s/%s/data/part-%d.parquet", namespaceName, tableName, time.Now().UnixNano())
createResp, err := s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("CreateMultipartUpload failed: %v", err)
}
uploadID := aws.ToString(createResp.UploadId)
multipartCompleted := false
defer func() {
if uploadID == "" || multipartCompleted {
return
}
_, _ = s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
})
}()
partSize := 5 * 1024 * 1024
part1 := bytes.Repeat([]byte("a"), partSize)
part2 := bytes.Repeat([]byte("b"), 1024*1024)
part1Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(1),
Body: bytes.NewReader(part1),
})
if err != nil {
t.Fatalf("UploadPart 1 failed: %v", err)
}
part2Resp, err := s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(2),
Body: bytes.NewReader(part2),
})
if err != nil {
t.Fatalf("UploadPart 2 failed: %v", err)
}
_, err = s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
MultipartUpload: &s3types.CompletedMultipartUpload{
Parts: []s3types.CompletedPart{
{
ETag: part1Resp.ETag,
PartNumber: aws.Int32(1),
},
{
ETag: part2Resp.ETag,
PartNumber: aws.Int32(2),
},
},
},
})
if err != nil {
t.Fatalf("CompleteMultipartUpload failed: %v", err)
}
multipartCompleted = true
headResp, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("HeadObject after multipart upload failed: %v", err)
}
expectedSize := int64(len(part1) + len(part2))
if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
}
if err := client.DeleteTable(ctx, bucketARN, namespaceName, tableName); err != nil {
t.Fatalf("DeleteTable failed: %v", err)
}
tableCreated = false
if err := client.DeleteNamespace(ctx, bucketARN, namespaceName); err != nil {
t.Fatalf("DeleteNamespace failed: %v", err)
}
namespaceCreated = false
if err := client.DeleteTableBucket(ctx, bucketARN); err != nil {
t.Fatalf("DeleteTableBucket failed: %v", err)
}
bucketDeleted = true
if _, err := client.GetTableBucket(ctx, bucketARN); err == nil {
t.Fatalf("expected GetTableBucket to fail after deletion")
}
}
type s3TablesClient struct {
endpoint string
region string
creds aws.Credentials
httpClient *http.Client
}
func newS3TablesClient(endpoint, region string, creds aws.Credentials) *s3TablesClient {
return &s3TablesClient{
endpoint: endpoint,
region: region,
creds: creds,
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
func (c *s3TablesClient) CreateTableBucket(ctx context.Context, name string) (string, error) {
req := &s3tables.CreateTableBucketRequest{Name: name}
var resp s3tables.CreateTableBucketResponse
if err := c.doRequest(ctx, "CreateTableBucket", http.MethodPut, "/buckets", req, &resp); err != nil {
return "", err
}
return resp.ARN, nil
}
func (c *s3TablesClient) GetTableBucket(ctx context.Context, arn string) (*s3tables.GetTableBucketResponse, error) {
path := "/buckets/" + url.PathEscape(arn)
var resp s3tables.GetTableBucketResponse
if err := c.doRequest(ctx, "GetTableBucket", http.MethodGet, path, nil, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func (c *s3TablesClient) ListTableBuckets(ctx context.Context) ([]s3tables.TableBucketSummary, error) {
var resp s3tables.ListTableBucketsResponse
if err := c.doRequest(ctx, "ListTableBuckets", http.MethodGet, "/buckets", nil, &resp); err != nil {
return nil, err
}
return resp.TableBuckets, nil
}
func (c *s3TablesClient) DeleteTableBucket(ctx context.Context, arn string) error {
path := "/buckets/" + url.PathEscape(arn)
return c.doRequest(ctx, "DeleteTableBucket", http.MethodDelete, path, nil, nil)
}
func (c *s3TablesClient) CreateNamespace(ctx context.Context, bucketARN, namespace string) error {
req := &s3tables.CreateNamespaceRequest{
Namespace: []string{namespace},
}
path := "/namespaces/" + url.PathEscape(bucketARN)
return c.doRequest(ctx, "CreateNamespace", http.MethodPut, path, req, nil)
}
func (c *s3TablesClient) DeleteNamespace(ctx context.Context, bucketARN, namespace string) error {
path := "/namespaces/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
return c.doRequest(ctx, "DeleteNamespace", http.MethodDelete, path, nil, nil)
}
func (c *s3TablesClient) CreateTable(ctx context.Context, bucketARN, namespace, name string) error {
req := &s3tables.CreateTableRequest{
Name: name,
Format: "ICEBERG",
}
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace)
return c.doRequest(ctx, "CreateTable", http.MethodPut, path, req, nil)
}
func (c *s3TablesClient) DeleteTable(ctx context.Context, bucketARN, namespace, name string) error {
path := "/tables/" + url.PathEscape(bucketARN) + "/" + url.PathEscape(namespace) + "/" + url.PathEscape(name)
return c.doRequest(ctx, "DeleteTable", http.MethodDelete, path, nil, nil)
}
func (c *s3TablesClient) doRequest(ctx context.Context, operation, method, path string, body interface{}, out interface{}) error {
var bodyBytes []byte
if body != nil {
encoded, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("%s: marshal request: %w", operation, err)
}
bodyBytes = encoded
}
req, err := http.NewRequestWithContext(ctx, method, c.endpoint+path, bytes.NewReader(bodyBytes))
if err != nil {
return fmt.Errorf("%s: create request: %w", operation, err)
}
if body != nil {
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
}
req.Host = req.URL.Host
req.Header.Set("Host", req.URL.Host)
payloadHash := sha256.Sum256(bodyBytes)
if err := v4.NewSigner().SignHTTP(ctx, c.creds, req, hex.EncodeToString(payloadHash[:]), "s3tables", c.region, time.Now()); err != nil {
return fmt.Errorf("%s: sign request: %w", operation, err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("%s: request failed: %w", operation, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
bodyBytes, readErr := io.ReadAll(resp.Body)
if readErr != nil {
return fmt.Errorf("%s failed with status %d and could not read response body: %v", operation, resp.StatusCode, readErr)
}
var errResp s3tables.S3TablesError
if jsonErr := json.Unmarshal(bodyBytes, &errResp); jsonErr == nil && (errResp.Type != "" || errResp.Message != "") {
return fmt.Errorf("%s failed: %s - %s", operation, errResp.Type, errResp.Message)
}
return fmt.Errorf("%s failed with status %d: %s", operation, resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
}
if out == nil {
return nil
}
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
return fmt.Errorf("%s: decode response: %w", operation, err)
}
return nil
}
func assumeRole(ctx context.Context, endpoint, region, accessKey, secretKey, roleArn, sessionName string) (aws.Credentials, error) {
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == sts.ServiceID {
return aws.Endpoint{
URL: endpoint,
SigningRegion: region,
HostnameImmutable: true,
}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, "")),
config.WithEndpointResolverWithOptions(resolver),
cmd := exec.CommandContext(dockerCtx, "docker", "run", "--rm",
"--name", containerName,
"--add-host", "host.docker.internal:host-gateway",
"-v", fmt.Sprintf("%s:/work", env.dataDir),
"python:3",
"/bin/bash", "-c", "pip install boto3 && python /work/lakekeeper_repro.py",
)
if err != nil {
return aws.Credentials{}, err
}
client := sts.NewFromConfig(cfg)
resp, err := client.AssumeRole(ctx, &sts.AssumeRoleInput{
RoleArn: aws.String(roleArn),
RoleSessionName: aws.String(sessionName),
})
output, err := cmd.CombinedOutput()
if err != nil {
return aws.Credentials{}, err
}
if resp.Credentials == nil {
return aws.Credentials{}, fmt.Errorf("missing credentials in AssumeRole response")
}
return aws.Credentials{
AccessKeyID: aws.ToString(resp.Credentials.AccessKeyId),
SecretAccessKey: aws.ToString(resp.Credentials.SecretAccessKey),
SessionToken: aws.ToString(resp.Credentials.SessionToken),
Source: "lakekeeper-sts",
}, nil
}
func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials) (*s3.Client, error) {
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID {
return aws.Endpoint{
URL: endpoint,
SigningRegion: region,
HostnameImmutable: true,
}, nil
if dockerCtx.Err() == context.DeadlineExceeded {
t.Fatalf("Lakekeeper repro client timed out after 5 minutes\nOutput:\n%s", string(output))
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
config.WithEndpointResolverWithOptions(resolver),
)
if err != nil {
return nil, err
t.Fatalf("Lakekeeper repro client failed: %v\nOutput:\n%s", err, string(output))
}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = true
}), nil
}
func newLakekeeperSession(ctx context.Context, env *TestEnvironment) (lakekeeperSession, error) {
endpoint := fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
creds, err := assumeRole(ctx, endpoint, lakekeeperRegion, env.accessKey, env.secretKey, lakekeeperRoleArn, lakekeeperSessionName)
if err != nil {
return lakekeeperSession{}, err
}
return lakekeeperSession{
endpoint: endpoint,
region: lakekeeperRegion,
creds: creds,
}, nil
t.Logf("Lakekeeper repro client output:\n%s", string(output))
}
-377
View File
@@ -1,377 +0,0 @@
package polaris
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/seaweedfs/seaweedfs/test/s3tables/testutil"
)
const (
polarisImage = "apache/polaris:latest"
polarisRealm = "POLARIS"
polarisRootClientID = "root"
polarisRootClientSecret = "s3cr3t"
polarisRegion = "us-east-1"
polarisRoleArn = "arn:aws:iam::000000000000:role/PolarisVendedRole"
polarisSigningKey = "dGVzdC1zaWduaW5nLWtleS1mb3Itc3RzLWludGVncmF0aW9uLXRlc3Rz" // gitleaks:allow - test signing key
)
type TestEnvironment struct {
seaweedDir string
weedBinary string
dataDir string
bindIP string
s3Port int
s3GrpcPort int
masterPort int
masterGrpcPort int
filerPort int
filerGrpcPort int
volumePort int
volumeGrpcPort int
polarisPort int
polarisAdminPort int
weedProcess *exec.Cmd
weedCancel context.CancelFunc
polarisContainer string
accessKey string
secretKey string
}
func NewTestEnvironment(t *testing.T) *TestEnvironment {
t.Helper()
if !testutil.HasDocker() {
t.Skip("Docker is required for Polaris integration tests")
}
wd, err := os.Getwd()
if err != nil {
t.Fatalf("Failed to get working directory: %v", err)
}
seaweedDir := wd
for i := 0; i < 6; i++ {
if _, err := os.Stat(filepath.Join(seaweedDir, "go.mod")); err == nil {
break
}
seaweedDir = filepath.Dir(seaweedDir)
}
weedBinary := filepath.Join(seaweedDir, "weed", "weed")
if _, err := os.Stat(weedBinary); err != nil {
weedBinary = "weed"
if _, err := exec.LookPath(weedBinary); err != nil {
t.Skip("weed binary not found, skipping integration test")
}
}
dataDir, err := os.MkdirTemp("", "seaweed-polaris-test-*")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
bindIP := testutil.FindBindIP()
masterPort, masterGrpcPort := testutil.MustFreePortPair(t, "Master")
volumePort, volumeGrpcPort := testutil.MustFreePortPair(t, "Volume")
filerPort, filerGrpcPort := testutil.MustFreePortPair(t, "Filer")
s3Port, s3GrpcPort := testutil.MustFreePortPair(t, "S3")
polarisPort, polarisAdminPort := testutil.MustFreePortPair(t, "Polaris")
return &TestEnvironment{
seaweedDir: seaweedDir,
weedBinary: weedBinary,
dataDir: dataDir,
bindIP: bindIP,
s3Port: s3Port,
s3GrpcPort: s3GrpcPort,
masterPort: masterPort,
masterGrpcPort: masterGrpcPort,
filerPort: filerPort,
filerGrpcPort: filerGrpcPort,
volumePort: volumePort,
volumeGrpcPort: volumeGrpcPort,
polarisPort: polarisPort,
polarisAdminPort: polarisAdminPort,
accessKey: "admin",
secretKey: "admin",
}
}
func (env *TestEnvironment) StartSeaweedFS(t *testing.T) {
t.Helper()
iamConfigPath := filepath.Join(env.dataDir, "iam.json")
iamConfig := fmt.Sprintf(`{
"identities": [
{
"name": "admin",
"credentials": [
{
"accessKey": "%s",
"secretKey": "%s"
}
],
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
}
],
"sts": {
"tokenDuration": "12h",
"maxSessionLength": "24h",
"issuer": "seaweedfs-sts",
"signingKey": "%s"
},
"roles": [
{
"roleName": "PolarisVendedRole",
"roleArn": "%s",
"trustPolicy": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": "*",
"Action": "sts:AssumeRole"
}
]
},
"attachedPolicies": ["FullAccess"]
}
],
"policies": [
{
"name": "FullAccess",
"document": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "*",
"Resource": "*"
}
]
}
}
]
}`, env.accessKey, env.secretKey, polarisSigningKey, polarisRoleArn)
if err := os.WriteFile(iamConfigPath, []byte(iamConfig), 0644); err != nil {
t.Fatalf("Failed to create IAM config: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
env.weedCancel = cancel
cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini",
"-master.port", fmt.Sprintf("%d", env.masterPort),
"-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort),
"-volume.port", fmt.Sprintf("%d", env.volumePort),
"-volume.port.grpc", fmt.Sprintf("%d", env.volumeGrpcPort),
"-filer.port", fmt.Sprintf("%d", env.filerPort),
"-filer.port.grpc", fmt.Sprintf("%d", env.filerGrpcPort),
"-s3.port", fmt.Sprintf("%d", env.s3Port),
"-s3.port.grpc", fmt.Sprintf("%d", env.s3GrpcPort),
"-s3.config", iamConfigPath,
"-s3.iam.config", iamConfigPath,
"-s3.iam.readOnly=false",
"-ip", env.bindIP,
"-ip.bind", "0.0.0.0",
"-dir", env.dataDir,
)
cmd.Dir = env.dataDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Start(); err != nil {
t.Fatalf("Failed to start SeaweedFS: %v", err)
}
env.weedProcess = cmd
if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/status", env.s3Port), 30*time.Second) {
t.Fatalf("S3 API failed to become ready")
}
}
func (env *TestEnvironment) StartPolaris(t *testing.T) {
t.Helper()
containerName := fmt.Sprintf("seaweed-polaris-%d", time.Now().UnixNano())
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
defer cancel()
cmd := exec.CommandContext(ctx, "docker", "run", "-d", "--rm",
"--name", containerName,
"--add-host", "host.docker.internal:host-gateway",
"-p", fmt.Sprintf("%d:8181", env.polarisPort),
"-p", fmt.Sprintf("%d:8182", env.polarisAdminPort),
"-e", fmt.Sprintf("AWS_REGION=%s", polarisRegion),
"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", env.accessKey),
"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", env.secretKey),
"-e", fmt.Sprintf("POLARIS_BOOTSTRAP_CREDENTIALS=%s,%s,%s", polarisRealm, polarisRootClientID, polarisRootClientSecret),
"-e", fmt.Sprintf("polaris.realm-context.realms=%s", polarisRealm),
"-e", "quarkus.otel.sdk.disabled=true",
polarisImage,
)
output, err := cmd.CombinedOutput()
if err != nil {
if ctx.Err() == context.DeadlineExceeded {
t.Fatalf("Timed out waiting for Polaris container: %v\nOutput:\n%s", ctx.Err(), string(output))
}
t.Fatalf("Failed to start Polaris: %v\nOutput:\n%s", err, string(output))
}
env.polarisContainer = containerName
if !testutil.WaitForService(fmt.Sprintf("http://localhost:%d/q/health", env.polarisAdminPort), 60*time.Second) {
logCtx, logCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer logCancel()
logs, _ := exec.CommandContext(logCtx, "docker", "logs", env.polarisContainer).CombinedOutput()
t.Fatalf("Polaris failed to become ready\nLogs:\n%s", string(logs))
}
}
func (env *TestEnvironment) Cleanup(t *testing.T) {
t.Helper()
if env.weedCancel != nil {
env.weedCancel()
}
if env.weedProcess != nil {
time.Sleep(1 * time.Second)
_ = env.weedProcess.Wait()
}
if env.polarisContainer != "" {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
_ = exec.CommandContext(ctx, "docker", "rm", "-f", env.polarisContainer).Run()
}
if env.dataDir != "" {
_ = os.RemoveAll(env.dataDir)
}
}
func (env *TestEnvironment) polarisEndpoint() string {
return fmt.Sprintf("http://127.0.0.1:%d", env.polarisPort)
}
func (env *TestEnvironment) s3Endpoint() string {
return fmt.Sprintf("http://127.0.0.1:%d", env.s3Port)
}
func (env *TestEnvironment) s3InternalEndpoint() string {
return fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
}
type polarisHTTPClient struct {
baseURL string
realm string
token string
httpClient *http.Client
}
func newPolarisHTTPClient(baseURL, realm, token string) *polarisHTTPClient {
return &polarisHTTPClient{
baseURL: baseURL,
realm: realm,
token: token,
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
func (c *polarisHTTPClient) doJSON(ctx context.Context, method, path string, body interface{}, out interface{}) error {
return c.doJSONWithHeaders(ctx, method, path, body, out, nil)
}
func (c *polarisHTTPClient) doJSONWithHeaders(ctx context.Context, method, path string, body interface{}, out interface{}, headers map[string]string) error {
var reader io.Reader
if body != nil {
encoded, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("encode request body: %w", err)
}
reader = bytes.NewReader(encoded)
}
req, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, reader)
if err != nil {
return fmt.Errorf("create request: %w", err)
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
req.Header.Set("Accept", "application/json")
if c.realm != "" {
req.Header.Set("Polaris-Realm", c.realm)
}
if c.token != "" {
req.Header.Set("Authorization", "Bearer "+c.token)
}
for key, value := range headers {
req.Header.Set(key, value)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
bodyBytes, readErr := io.ReadAll(resp.Body)
if readErr != nil {
return fmt.Errorf("request failed with status %d and could not read response body: %w", resp.StatusCode, readErr)
}
return fmt.Errorf("request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(bodyBytes)))
}
if out == nil {
return nil
}
if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
return fmt.Errorf("decode response: %w", err)
}
return nil
}
func newS3Client(ctx context.Context, endpoint, region string, creds aws.Credentials, pathStyle bool) (*s3.Client, error) {
resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID {
return aws.Endpoint{
URL: endpoint,
SigningRegion: region,
HostnameImmutable: true,
}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(creds.AccessKeyID, creds.SecretAccessKey, creds.SessionToken)),
config.WithEndpointResolverWithOptions(resolver),
)
if err != nil {
return nil, err
}
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = pathStyle
}), nil
}
-764
View File
@@ -1,764 +0,0 @@
package polaris
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
)
type polarisSession struct {
catalogName string
bucketName string
token string
baseLocation string
}
type polarisTableSetup struct {
namespace string
table string
dataKeyPrefix string
s3Client *s3.Client
}
type polarisCatalogClient struct {
http *polarisHTTPClient
catalog string
}
type polarisCredentials struct {
ClientID string `json:"clientId"`
ClientSecret string `json:"clientSecret"`
}
type createPrincipalResponse struct {
Credentials polarisCredentials `json:"credentials"`
}
type createCatalogRequest struct {
Catalog polarisCatalog `json:"catalog"`
}
type polarisCatalog struct {
Name string `json:"name"`
Type string `json:"type"`
ReadOnly bool `json:"readOnly"`
Properties map[string]string `json:"properties"`
StorageConfigInfo polarisStorageConfig `json:"storageConfigInfo"`
}
type polarisStorageConfig struct {
StorageType string `json:"storageType"`
AllowedLocations []string `json:"allowedLocations"`
Endpoint string `json:"endpoint"`
EndpointInternal string `json:"endpointInternal"`
StsEndpoint string `json:"stsEndpoint"`
PathStyleAccess bool `json:"pathStyleAccess"`
RoleArn string `json:"roleArn"`
Region string `json:"region"`
}
type createNamespaceRequest struct {
Namespace []string `json:"namespace"`
}
type createTableRequest struct {
Name string `json:"name"`
Location string `json:"location"`
Schema icebergSchema `json:"schema"`
PartitionSpec icebergPartition `json:"partition-spec"`
SortOrder icebergSortOrder `json:"sort-order"`
Properties map[string]string `json:"properties"`
}
type icebergSchema struct {
Type string `json:"type"`
SchemaID int `json:"schema-id"`
Fields []icebergSchemaField `json:"fields"`
}
type icebergSchemaField struct {
ID int `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
Required bool `json:"required"`
}
type icebergPartition struct {
SpecID int `json:"spec-id"`
Fields []icebergPartitionField `json:"fields"`
}
type icebergPartitionField struct {
SourceID int `json:"source-id"`
FieldID int `json:"field-id"`
Name string `json:"name"`
Transform string `json:"transform"`
}
type icebergSortOrder struct {
OrderID int `json:"order-id"`
Fields []icebergSortField `json:"fields"`
}
type icebergSortField struct {
SourceID int `json:"source-id"`
Direction string `json:"direction"`
NullOrder string `json:"null-order"`
}
type loadTableResponse struct {
Config map[string]string `json:"config"`
StorageCredentials []storageCredential `json:"storage-credentials"`
}
type loadCredentialsResponse struct {
StorageCredentials []storageCredential `json:"storage-credentials"`
}
type storageCredential struct {
Prefix string `json:"prefix"`
Config map[string]string `json:"config"`
}
func bootstrapPolarisTest(t *testing.T, env *TestEnvironment) (context.Context, context.CancelFunc, polarisSession, *polarisTableSetup, func()) {
t.Helper()
t.Logf(">>> Starting SeaweedFS with Polaris configuration...")
env.StartSeaweedFS(t)
t.Logf(">>> SeaweedFS started.")
t.Logf(">>> Starting Polaris...")
env.StartPolaris(t)
t.Logf(">>> Polaris started.")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
session := newPolarisSession(t, ctx, env)
setup, cleanup := setupPolarisTable(t, ctx, env, session)
return ctx, cancel, session, setup, cleanup
}
func TestPolarisIntegration(t *testing.T) {
env := NewTestEnvironment(t)
defer env.Cleanup(t)
ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
defer cancel()
defer cleanup()
objectKey := fmt.Sprintf("%s/hello-%d.txt", setup.dataKeyPrefix, time.Now().UnixNano())
payload := []byte("polaris")
if _, err := setup.s3Client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
Body: bytes.NewReader(payload),
}); err != nil {
t.Fatalf("PutObject failed: %v", err)
}
listObjects, err := setup.s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(session.bucketName),
Prefix: aws.String(objectKey),
})
if err != nil {
t.Fatalf("ListObjectsV2 failed: %v", err)
}
found := false
for _, obj := range listObjects.Contents {
if aws.ToString(obj.Key) == objectKey {
found = true
break
}
}
if !found {
t.Fatalf("Object %s not found in list", objectKey)
}
getResp, err := setup.s3Client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("GetObject failed: %v", err)
}
body, err := io.ReadAll(getResp.Body)
_ = getResp.Body.Close()
if err != nil {
t.Fatalf("Read object body failed: %v", err)
}
if !bytes.Equal(body, payload) {
t.Fatalf("Unexpected object payload: got %q want %q", string(body), string(payload))
}
if _, err := setup.s3Client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
}); err != nil {
t.Fatalf("DeleteObject failed: %v", err)
}
}
func TestPolarisTableIntegration(t *testing.T) {
env := NewTestEnvironment(t)
defer env.Cleanup(t)
ctx, cancel, session, setup, cleanup := bootstrapPolarisTest(t, env)
defer cancel()
defer cleanup()
objectKey := fmt.Sprintf("%s/part-%d.parquet", setup.dataKeyPrefix, time.Now().UnixNano())
createResp, err := setup.s3Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("CreateMultipartUpload failed: %v", err)
}
uploadID := aws.ToString(createResp.UploadId)
multipartCompleted := false
defer func() {
if uploadID == "" || multipartCompleted {
return
}
_, _ = setup.s3Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
})
}()
partSize := 5 * 1024 * 1024
part1 := bytes.Repeat([]byte("a"), partSize)
part2 := bytes.Repeat([]byte("b"), 1024*1024)
part1Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(1),
Body: bytes.NewReader(part1),
})
if err != nil {
t.Fatalf("UploadPart 1 failed: %v", err)
}
part2Resp, err := setup.s3Client.UploadPart(ctx, &s3.UploadPartInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
PartNumber: aws.Int32(2),
Body: bytes.NewReader(part2),
})
if err != nil {
t.Fatalf("UploadPart 2 failed: %v", err)
}
_, err = setup.s3Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
UploadId: aws.String(uploadID),
MultipartUpload: &s3types.CompletedMultipartUpload{
Parts: []s3types.CompletedPart{
{
ETag: part1Resp.ETag,
PartNumber: aws.Int32(1),
},
{
ETag: part2Resp.ETag,
PartNumber: aws.Int32(2),
},
},
},
})
if err != nil {
t.Fatalf("CompleteMultipartUpload failed: %v", err)
}
multipartCompleted = true
headResp, err := setup.s3Client.HeadObject(ctx, &s3.HeadObjectInput{
Bucket: aws.String(session.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
t.Fatalf("HeadObject after multipart upload failed: %v", err)
}
expectedSize := int64(len(part1) + len(part2))
if headResp.ContentLength == nil || *headResp.ContentLength != expectedSize {
t.Fatalf("Unexpected content length: got %d want %d", aws.ToInt64(headResp.ContentLength), expectedSize)
}
}
func newPolarisSession(t *testing.T, ctx context.Context, env *TestEnvironment) polarisSession {
t.Helper()
adminCreds := aws.Credentials{
AccessKeyID: env.accessKey,
SecretAccessKey: env.secretKey,
Source: "polaris-admin",
}
adminS3, err := newS3Client(ctx, env.s3Endpoint(), polarisRegion, adminCreds, true)
if err != nil {
t.Fatalf("Create admin S3 client failed: %v", err)
}
bucketName := fmt.Sprintf("polaris-bucket-%d", time.Now().UnixNano())
if _, err := adminS3.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: aws.String(bucketName),
}); err != nil {
t.Fatalf("CreateBucket failed: %v", err)
}
policyDoc := map[string]interface{}{
"Version": "2012-10-17",
"Statement": []map[string]interface{}{
{
"Sid": "AllowPolarisVendedAccess",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:*",
"Resource": []string{
fmt.Sprintf("arn:aws:s3:::%s", bucketName),
fmt.Sprintf("arn:aws:s3:::%s/polaris/*", bucketName),
},
},
},
}
policyBytes, err := json.Marshal(policyDoc)
if err != nil {
t.Fatalf("Failed to marshal bucket policy: %v", err)
}
if _, err := adminS3.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
Bucket: aws.String(bucketName),
Policy: aws.String(string(policyBytes)),
}); err != nil {
t.Fatalf("PutBucketPolicy failed: %v", err)
}
rootToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), polarisRootClientID, polarisRootClientSecret)
if err != nil {
t.Fatalf("Polaris root token request failed: %v", err)
}
managementClient := newPolarisHTTPClient(env.polarisEndpoint(), polarisRealm, rootToken)
catalogName := fmt.Sprintf("polaris_catalog_%d", time.Now().UnixNano())
baseLocation := fmt.Sprintf("s3://%s/polaris", bucketName)
catalogRequest := createCatalogRequest{
Catalog: polarisCatalog{
Name: catalogName,
Type: "INTERNAL",
ReadOnly: false,
Properties: map[string]string{
"default-base-location": baseLocation,
},
StorageConfigInfo: polarisStorageConfig{
StorageType: "S3",
AllowedLocations: []string{baseLocation},
Endpoint: env.s3Endpoint(),
EndpointInternal: env.s3InternalEndpoint(),
StsEndpoint: env.s3InternalEndpoint(),
PathStyleAccess: true,
RoleArn: polarisRoleArn,
Region: polarisRegion,
},
},
}
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/catalogs", catalogRequest, nil); err != nil {
t.Fatalf("Create catalog failed: %v", err)
}
principalName := fmt.Sprintf("polaris_user_%d", time.Now().UnixNano())
principalRoleName := fmt.Sprintf("polaris_principal_role_%d", time.Now().UnixNano())
catalogRoleName := fmt.Sprintf("polaris_catalog_role_%d", time.Now().UnixNano())
var principalResp createPrincipalResponse
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principals", map[string]interface{}{
"principal": map[string]interface{}{
"name": principalName,
"properties": map[string]string{},
},
}, &principalResp); err != nil {
t.Fatalf("Create principal failed: %v", err)
}
if principalResp.Credentials.ClientID == "" || principalResp.Credentials.ClientSecret == "" {
t.Fatalf("Missing principal credentials in response")
}
if err := managementClient.doJSON(ctx, http.MethodPost, "/api/management/v1/principal-roles", map[string]interface{}{
"principalRole": map[string]interface{}{
"name": principalRoleName,
"properties": map[string]string{},
},
}, nil); err != nil {
t.Fatalf("Create principal role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPost, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles", url.PathEscape(catalogName)), map[string]interface{}{
"catalogRole": map[string]interface{}{
"name": catalogRoleName,
"properties": map[string]string{},
},
}, nil); err != nil {
t.Fatalf("Create catalog role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principals/%s/principal-roles", url.PathEscape(principalName)), map[string]interface{}{
"principalRole": map[string]interface{}{
"name": principalRoleName,
},
}, nil); err != nil {
t.Fatalf("Assign principal role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/principal-roles/%s/catalog-roles/%s", url.PathEscape(principalRoleName), url.PathEscape(catalogName)), map[string]interface{}{
"catalogRole": map[string]interface{}{
"name": catalogRoleName,
},
}, nil); err != nil {
t.Fatalf("Assign catalog role failed: %v", err)
}
if err := managementClient.doJSON(ctx, http.MethodPut, fmt.Sprintf("/api/management/v1/catalogs/%s/catalog-roles/%s/grants", url.PathEscape(catalogName), url.PathEscape(catalogRoleName)), map[string]interface{}{
"type": "catalog",
"privilege": "CATALOG_MANAGE_CONTENT",
}, nil); err != nil {
t.Fatalf("Grant catalog privilege failed: %v", err)
}
userToken, err := fetchPolarisToken(ctx, env.polarisEndpoint(), principalResp.Credentials.ClientID, principalResp.Credentials.ClientSecret)
if err != nil {
t.Fatalf("Polaris user token request failed: %v", err)
}
return polarisSession{
catalogName: catalogName,
bucketName: bucketName,
token: userToken,
baseLocation: baseLocation,
}
}
func setupPolarisTable(t *testing.T, ctx context.Context, env *TestEnvironment, session polarisSession) (*polarisTableSetup, func()) {
t.Helper()
catalogClient := newPolarisCatalogClient(env.polarisEndpoint(), polarisRealm, session.token, session.catalogName)
namespace := fmt.Sprintf("polaris_ns_%d", time.Now().UnixNano())
table := fmt.Sprintf("polaris_table_%d", time.Now().UnixNano())
if err := catalogClient.CreateNamespace(ctx, namespace); err != nil {
t.Fatalf("CreateNamespace failed: %v", err)
}
location := fmt.Sprintf("%s/%s/%s", session.baseLocation, namespace, table)
if err := catalogClient.CreateTable(ctx, namespace, table, location); err != nil {
t.Fatalf("CreateTable failed: %v", err)
}
loadResp, err := catalogClient.LoadTable(ctx, namespace, table)
if err != nil {
t.Fatalf("LoadTable failed: %v", err)
}
credsResp, err := catalogClient.LoadCredentials(ctx, namespace, table)
if err != nil {
t.Fatalf("LoadCredentials failed: %v", err)
}
if len(credsResp.StorageCredentials) == 0 {
t.Fatalf("LoadCredentials returned no storage credentials")
}
credentialSource := &loadTableResponse{
Config: loadResp.Config,
StorageCredentials: credsResp.StorageCredentials,
}
dataPrefix := location + "/data"
creds, endpoint, region, pathStyle, err := extractS3Credentials(credentialSource, dataPrefix, env.s3Endpoint(), polarisRegion)
if err != nil {
t.Fatalf("Extract vended credentials failed: %v", err)
}
dataKeyPrefix, err := s3URIToKeyPrefix(dataPrefix, session.bucketName)
if err != nil {
t.Fatalf("Invalid data prefix %s: %v", dataPrefix, err)
}
s3Client, err := newS3Client(ctx, endpoint, region, creds, pathStyle)
if err != nil {
t.Fatalf("Create vended S3 client failed: %v", err)
}
cleanup := func() {
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cleanupCancel()
if err := catalogClient.DeleteTable(cleanupCtx, namespace, table); err != nil {
t.Logf("DeleteTable failed: %v", err)
}
if err := catalogClient.DeleteNamespace(cleanupCtx, namespace); err != nil {
t.Logf("DeleteNamespace failed: %v", err)
}
}
return &polarisTableSetup{
namespace: namespace,
table: table,
dataKeyPrefix: dataKeyPrefix,
s3Client: s3Client,
}, cleanup
}
func fetchPolarisToken(ctx context.Context, baseURL, clientID, clientSecret string) (string, error) {
form := url.Values{}
form.Set("grant_type", "client_credentials")
form.Set("client_id", clientID)
form.Set("client_secret", clientSecret)
form.Set("scope", "PRINCIPAL_ROLE:ALL")
req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL+"/api/catalog/v1/oauth/tokens", strings.NewReader(form.Encode()))
if err != nil {
return "", fmt.Errorf("create token request: %w", err)
}
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
req.Header.Set("Accept", "application/json")
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("token request failed: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
body, readErr := io.ReadAll(resp.Body)
if readErr != nil {
return "", fmt.Errorf("token request failed with status %d and reading body: %w", resp.StatusCode, readErr)
}
return "", fmt.Errorf("token request failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}
var tokenResp struct {
AccessToken string `json:"access_token"`
}
if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
return "", fmt.Errorf("decode token response: %w", err)
}
if tokenResp.AccessToken == "" {
return "", fmt.Errorf("missing access token in response")
}
return tokenResp.AccessToken, nil
}
func newPolarisCatalogClient(baseURL, realm, token, catalog string) *polarisCatalogClient {
return &polarisCatalogClient{
http: newPolarisHTTPClient(baseURL, realm, token),
catalog: catalog,
}
}
func (c *polarisCatalogClient) CreateNamespace(ctx context.Context, namespace string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces", url.PathEscape(c.catalog))
req := createNamespaceRequest{Namespace: []string{namespace}}
return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
}
func (c *polarisCatalogClient) DeleteNamespace(ctx context.Context, namespace string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s", url.PathEscape(c.catalog), url.PathEscape(namespace))
return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
}
func (c *polarisCatalogClient) CreateTable(ctx context.Context, namespace, table, location string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables", url.PathEscape(c.catalog), url.PathEscape(namespace))
req := createTableRequest{
Name: table,
Location: location,
Schema: icebergSchema{
Type: "struct",
SchemaID: 0,
Fields: []icebergSchemaField{
{
ID: 1,
Name: "id",
Type: "long",
Required: false,
},
},
},
PartitionSpec: icebergPartition{
SpecID: 0,
Fields: []icebergPartitionField{},
},
SortOrder: icebergSortOrder{
OrderID: 0,
Fields: []icebergSortField{},
},
Properties: map[string]string{
"format-version": "2",
},
}
return c.http.doJSON(ctx, http.MethodPost, path, req, nil)
}
func (c *polarisCatalogClient) DeleteTable(ctx context.Context, namespace, table string) error {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
return c.http.doJSON(ctx, http.MethodDelete, path, nil, nil)
}
func (c *polarisCatalogClient) LoadTable(ctx context.Context, namespace, table string) (*loadTableResponse, error) {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
var resp loadTableResponse
headers := map[string]string{
"X-Iceberg-Access-Delegation": "vended-credentials",
}
if err := c.http.doJSONWithHeaders(ctx, http.MethodGet, path, nil, &resp, headers); err != nil {
return nil, err
}
return &resp, nil
}
func (c *polarisCatalogClient) LoadCredentials(ctx context.Context, namespace, table string) (*loadCredentialsResponse, error) {
path := fmt.Sprintf("/api/catalog/v1/%s/namespaces/%s/tables/%s/credentials", url.PathEscape(c.catalog), url.PathEscape(namespace), url.PathEscape(table))
var resp loadCredentialsResponse
if err := c.http.doJSON(ctx, http.MethodGet, path, nil, &resp); err != nil {
return nil, err
}
return &resp, nil
}
func extractS3Credentials(load *loadTableResponse, targetPrefix, fallbackEndpoint, fallbackRegion string) (aws.Credentials, string, string, bool, error) {
credentialConfig, err := selectStorageConfig(load, targetPrefix)
if err != nil {
return aws.Credentials{}, "", "", false, err
}
lookupConfig := func(key string) string {
if load != nil && load.Config != nil {
if val, ok := load.Config[key]; ok && strings.TrimSpace(val) != "" {
return strings.TrimSpace(val)
}
}
if val, ok := credentialConfig[key]; ok && strings.TrimSpace(val) != "" {
return strings.TrimSpace(val)
}
return ""
}
accessKey := strings.TrimSpace(credentialConfig["s3.access-key-id"])
secretKey := strings.TrimSpace(credentialConfig["s3.secret-access-key"])
sessionToken := strings.TrimSpace(credentialConfig["s3.session-token"])
if accessKey == "" || secretKey == "" {
return aws.Credentials{}, "", "", false, fmt.Errorf("missing s3.access-key-id or s3.secret-access-key in selected storage credential")
}
endpoint := lookupConfig("s3.endpoint")
if endpoint == "" {
endpoint = fallbackEndpoint
}
if endpoint != "" && !strings.HasPrefix(endpoint, "http") {
endpoint = "http://" + endpoint
}
region := lookupConfig("client.region")
if region == "" {
region = fallbackRegion
}
pathStyle := true
if value := lookupConfig("s3.path-style-access"); value != "" {
pathStyle = strings.EqualFold(value, "true")
}
return aws.Credentials{
AccessKeyID: accessKey,
SecretAccessKey: secretKey,
SessionToken: sessionToken,
Source: "polaris-vended",
}, endpoint, region, pathStyle, nil
}
func selectStorageConfig(load *loadTableResponse, targetPrefix string) (map[string]string, error) {
if load == nil {
return nil, fmt.Errorf("load table response is nil")
}
switch len(load.StorageCredentials) {
case 0:
if load.Config == nil {
return nil, fmt.Errorf("polaris returned no storage credentials or config")
}
return load.Config, nil
case 1:
cred := load.StorageCredentials[0]
if cred.Config == nil {
return nil, fmt.Errorf("storage credential for prefix %s returned nil config", cred.Prefix)
}
return cred.Config, nil
default:
if targetPrefix == "" {
return nil, fmt.Errorf("multiple storage credentials (%d) returned but no target prefix provided", len(load.StorageCredentials))
}
normalizedTarget := normalizePrefix(targetPrefix)
if normalizedTarget == "" {
return nil, fmt.Errorf("target prefix %q normalized to empty string", targetPrefix)
}
var bestConfig map[string]string
bestLen := -1
for _, cred := range load.StorageCredentials {
if cred.Config == nil {
continue
}
prefix := normalizePrefix(cred.Prefix)
if prefix == "" {
if bestLen < 0 {
bestLen = 0
bestConfig = cred.Config
}
continue
}
if normalizedTarget == prefix || strings.HasPrefix(normalizedTarget, prefix+"/") {
if len(prefix) > bestLen {
bestLen = len(prefix)
bestConfig = cred.Config
}
}
}
if bestConfig != nil {
return bestConfig, nil
}
return nil, fmt.Errorf("none of the %d storage credentials matched prefix %s", len(load.StorageCredentials), targetPrefix)
}
}
func normalizePrefix(prefix string) string {
p := strings.TrimSpace(prefix)
p = strings.TrimSuffix(p, "/")
return p
}
func s3URIToKeyPrefix(uri, bucket string) (string, error) {
prefix := "s3://" + bucket + "/"
if !strings.HasPrefix(uri, prefix) {
return "", fmt.Errorf("uri %q does not match bucket %q", uri, bucket)
}
keyPrefix := strings.TrimPrefix(uri, prefix)
keyPrefix = strings.TrimPrefix(keyPrefix, "/")
if keyPrefix == "" {
return "", fmt.Errorf("empty key prefix in uri %q", uri)
}
return keyPrefix, nil
}
-148
View File
@@ -1,148 +0,0 @@
package dash
import (
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/wdclient/exclusive_locks"
)
const (
adminLockName = "shell"
adminLockClientName = "admin-plugin"
)
// AdminLockManager coordinates exclusive admin locks with reference counting.
// It is safe for concurrent use.
type AdminLockManager struct {
locker *exclusive_locks.ExclusiveLocker
clientName string
mu sync.Mutex
cond *sync.Cond
acquiring bool
holdCount int
lastAcquiredAt time.Time
lastReleasedAt time.Time
waitingSince time.Time
waitingReason string
currentReason string
}
func NewAdminLockManager(masterClient *wdclient.MasterClient, clientName string) *AdminLockManager {
if masterClient == nil {
return nil
}
if clientName == "" {
clientName = adminLockClientName
}
manager := &AdminLockManager{
locker: exclusive_locks.NewExclusiveLocker(masterClient, adminLockName),
clientName: clientName,
}
manager.cond = sync.NewCond(&manager.mu)
return manager
}
func (m *AdminLockManager) Acquire(reason string) (func(), error) {
if m == nil || m.locker == nil {
return func() {}, nil
}
m.mu.Lock()
if reason != "" {
m.locker.SetMessage(reason)
m.currentReason = reason
}
for m.acquiring {
m.cond.Wait()
}
if m.holdCount == 0 {
m.acquiring = true
m.waitingSince = time.Now().UTC()
m.waitingReason = reason
m.mu.Unlock()
m.locker.RequestLock(m.clientName)
m.mu.Lock()
m.acquiring = false
m.holdCount = 1
m.lastAcquiredAt = time.Now().UTC()
m.waitingSince = time.Time{}
m.waitingReason = ""
m.cond.Broadcast()
m.mu.Unlock()
return m.Release, nil
}
m.holdCount++
if reason != "" {
m.currentReason = reason
}
m.mu.Unlock()
return m.Release, nil
}
func (m *AdminLockManager) Release() {
if m == nil || m.locker == nil {
return
}
m.mu.Lock()
if m.holdCount <= 0 {
m.mu.Unlock()
return
}
m.holdCount--
shouldRelease := m.holdCount == 0
m.mu.Unlock()
if shouldRelease {
m.mu.Lock()
m.lastReleasedAt = time.Now().UTC()
m.currentReason = ""
m.mu.Unlock()
m.locker.ReleaseLock()
}
}
type LockStatus struct {
Held bool `json:"held"`
HoldCount int `json:"hold_count"`
Acquiring bool `json:"acquiring"`
Message string `json:"message,omitempty"`
WaitingReason string `json:"waiting_reason,omitempty"`
LastAcquiredAt *time.Time `json:"last_acquired_at,omitempty"`
LastReleasedAt *time.Time `json:"last_released_at,omitempty"`
WaitingSince *time.Time `json:"waiting_since,omitempty"`
}
func (m *AdminLockManager) Status() LockStatus {
if m == nil {
return LockStatus{}
}
m.mu.Lock()
defer m.mu.Unlock()
status := LockStatus{
Held: m.holdCount > 0,
HoldCount: m.holdCount,
Acquiring: m.acquiring,
Message: m.currentReason,
WaitingReason: m.waitingReason,
}
if !m.lastAcquiredAt.IsZero() {
at := m.lastAcquiredAt
status.LastAcquiredAt = &at
}
if !m.lastReleasedAt.IsZero() {
at := m.lastReleasedAt
status.LastReleasedAt = &at
}
if !m.waitingSince.IsZero() {
at := m.waitingSince
status.WaitingSince = &at
}
return status
}
+3 -55
View File
@@ -98,8 +98,6 @@ type AdminServer struct {
// Maintenance system
maintenanceManager *maintenance.MaintenanceManager
plugin *adminplugin.Plugin
pluginLock *AdminLockManager
expireJobHandler func(jobID string, reason string) (*adminplugin.TrackedJob, bool, error)
// Topic retention purger
topicRetentionPurger *TopicRetentionPurger
@@ -118,7 +116,7 @@ type AdminServer struct {
// Type definitions moved to types.go
func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string, icebergPort int, idleSleepDuration time.Duration) *AdminServer {
func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string, icebergPort int) *AdminServer {
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.admin")
// Create master client with multiple master support
@@ -136,8 +134,6 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
ctx := context.Background()
go masterClient.KeepConnectedToMaster(ctx)
lockManager := NewAdminLockManager(masterClient, adminLockClientName)
server := &AdminServer{
masterClient: masterClient,
templateFS: templateFS,
@@ -149,7 +145,6 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
collectionStatsCacheThreshold: defaultStatsCacheTimeout,
s3TablesManager: newS3TablesManager(),
icebergPort: icebergPort,
pluginLock: lockManager,
}
// Initialize topic retention purger
@@ -229,22 +224,18 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
}
plugin, err := adminplugin.New(adminplugin.Options{
DataDir: dataDir,
IdleSleepDuration: idleSleepDuration,
DataDir: dataDir,
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
return server.buildDefaultPluginClusterContext(), nil
},
LockManager: lockManager,
})
if err != nil && dataDir != "" {
glog.Warningf("Failed to initialize plugin with dataDir=%q: %v. Falling back to in-memory plugin state.", dataDir, err)
plugin, err = adminplugin.New(adminplugin.Options{
DataDir: "",
IdleSleepDuration: idleSleepDuration,
DataDir: "",
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
return server.buildDefaultPluginClusterContext(), nil
},
LockManager: lockManager,
})
}
if err != nil {
@@ -898,13 +889,6 @@ func (s *AdminServer) GetPlugin() *adminplugin.Plugin {
return s.plugin
}
func (s *AdminServer) acquirePluginLock(reason string) (func(), error) {
if s == nil || s.pluginLock == nil {
return func() {}, nil
}
return s.pluginLock.Acquire(reason)
}
// RequestPluginJobTypeDescriptor asks one worker for job type schema and returns the descriptor.
func (s *AdminServer) RequestPluginJobTypeDescriptor(ctx context.Context, jobType string, forceRefresh bool) (*plugin_pb.JobTypeDescriptor, error) {
if s.plugin == nil {
@@ -947,13 +931,6 @@ func (s *AdminServer) RunPluginDetection(
if s.plugin == nil {
return nil, fmt.Errorf("plugin is not enabled")
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
if err != nil {
return nil, err
}
if releaseLock != nil {
defer releaseLock()
}
return s.plugin.RunDetection(ctx, jobType, clusterContext, maxResults)
}
@@ -979,13 +956,6 @@ func (s *AdminServer) RunPluginDetectionWithReport(
if s.plugin == nil {
return nil, fmt.Errorf("plugin is not enabled")
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detection %s", jobType))
if err != nil {
return nil, err
}
if releaseLock != nil {
defer releaseLock()
}
return s.plugin.RunDetectionWithReport(ctx, jobType, clusterContext, maxResults)
}
@@ -999,17 +969,6 @@ func (s *AdminServer) ExecutePluginJob(
if s.plugin == nil {
return nil, fmt.Errorf("plugin is not enabled")
}
jobType := ""
if job != nil {
jobType = strings.TrimSpace(job.JobType)
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin execution %s", jobType))
if err != nil {
return nil, err
}
if releaseLock != nil {
defer releaseLock()
}
return s.plugin.ExecuteJob(ctx, job, clusterContext, attempt)
}
@@ -1061,17 +1020,6 @@ func (s *AdminServer) GetPluginJobDetail(jobID string, activityLimit, relatedLim
return s.plugin.BuildJobDetail(jobID, activityLimit, relatedLimit)
}
// ExpirePluginJob marks an active plugin job as failed so it no longer blocks scheduling.
func (s *AdminServer) ExpirePluginJob(jobID, reason string) (*adminplugin.TrackedJob, bool, error) {
if handler := s.expireJobHandler; handler != nil {
return handler(jobID, reason)
}
if s.plugin == nil {
return nil, false, fmt.Errorf("plugin is not enabled")
}
return s.plugin.ExpireJob(jobID, reason)
}
// ListPluginActivities returns plugin job activities for monitoring.
func (s *AdminServer) ListPluginActivities(jobType string, limit int) []adminplugin.JobActivity {
if s.plugin == nil {
-158
View File
@@ -5,7 +5,6 @@ import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -131,47 +130,6 @@ func (s *AdminServer) GetPluginJobDetailAPI(w http.ResponseWriter, r *http.Reque
writeJSON(w, http.StatusOK, detail)
}
// ExpirePluginJobAPI marks a job as failed so it no longer blocks scheduling.
func (s *AdminServer) ExpirePluginJobAPI(w http.ResponseWriter, r *http.Request) {
jobID := strings.TrimSpace(mux.Vars(r)["jobId"])
if jobID == "" {
writeJSONError(w, http.StatusBadRequest, "jobId is required")
return
}
var req struct {
Reason string `json:"reason"`
}
if err := decodeJSONBody(newJSONMaxReader(w, r), &req); err != nil && err != io.EOF {
writeJSONError(w, http.StatusBadRequest, "invalid request body: "+err.Error())
return
}
job, expired, err := s.ExpirePluginJob(jobID, req.Reason)
if err != nil {
if errors.Is(err, plugin.ErrJobNotFound) {
writeJSONError(w, http.StatusNotFound, err.Error())
return
}
writeJSONError(w, http.StatusInternalServerError, err.Error())
return
}
response := map[string]interface{}{
"job_id": jobID,
"expired": expired,
}
if job != nil {
response["job"] = job
}
if !expired {
response["message"] = "job is not active"
}
writeJSON(w, http.StatusOK, response)
}
// GetPluginActivitiesAPI returns recent plugin activities.
func (s *AdminServer) GetPluginActivitiesAPI(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
@@ -214,27 +172,6 @@ func (s *AdminServer) GetPluginSchedulerStatesAPI(w http.ResponseWriter, r *http
writeJSON(w, http.StatusOK, states)
}
// GetPluginSchedulerStatusAPI returns scheduler status including in-process jobs and lock state.
func (s *AdminServer) GetPluginSchedulerStatusAPI(w http.ResponseWriter, r *http.Request) {
pluginSvc := s.GetPlugin()
if pluginSvc == nil {
writeJSON(w, http.StatusOK, map[string]interface{}{
"enabled": false,
})
return
}
response := map[string]interface{}{
"enabled": true,
"scheduler": pluginSvc.GetSchedulerStatus(),
}
if s.pluginLock != nil {
response["lock"] = s.pluginLock.Status()
}
writeJSON(w, http.StatusOK, response)
}
// RequestPluginJobTypeSchemaAPI asks a worker for one job type schema.
func (s *AdminServer) RequestPluginJobTypeSchemaAPI(w http.ResponseWriter, r *http.Request) {
jobType := strings.TrimSpace(mux.Vars(r)["jobType"])
@@ -298,9 +235,6 @@ func (s *AdminServer) GetPluginJobTypeConfigAPI(w http.ResponseWriter, r *http.R
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
}
}
if descriptor, err := s.LoadPluginJobTypeDescriptor(jobType); err == nil && descriptor != nil {
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
}
renderProtoJSON(w, http.StatusOK, config)
}
@@ -479,14 +413,6 @@ func (s *AdminServer) RunPluginJobTypeAPI(w http.ResponseWriter, r *http.Request
writeJSONError(w, http.StatusBadRequest, "jobType is required")
return
}
releaseLock, err := s.acquirePluginLock(fmt.Sprintf("plugin detect+execute %s", jobType))
if err != nil {
writeJSONError(w, http.StatusInternalServerError, err.Error())
return
}
if releaseLock != nil {
defer releaseLock()
}
var req struct {
ClusterContext json.RawMessage `json:"cluster_context"`
@@ -803,90 +729,6 @@ func buildJobSpecFromProposal(jobType string, proposal *plugin_pb.JobProposal, i
return jobSpec
}
func applyDescriptorDefaultsToPersistedConfig(
config *plugin_pb.PersistedJobTypeConfig,
descriptor *plugin_pb.JobTypeDescriptor,
) {
if config == nil || descriptor == nil {
return
}
if config.AdminConfigValues == nil {
config.AdminConfigValues = map[string]*plugin_pb.ConfigValue{}
}
if config.WorkerConfigValues == nil {
config.WorkerConfigValues = map[string]*plugin_pb.ConfigValue{}
}
if config.AdminRuntime == nil {
config.AdminRuntime = &plugin_pb.AdminRuntimeConfig{}
}
if descriptor.AdminConfigForm != nil {
for key, value := range descriptor.AdminConfigForm.DefaultValues {
if value == nil {
continue
}
current := config.AdminConfigValues[key]
if current == nil {
config.AdminConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
continue
}
if strings.EqualFold(descriptor.JobType, "admin_script") &&
key == "script" &&
isBlankStringConfigValue(current) {
config.AdminConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
}
}
}
if descriptor.WorkerConfigForm != nil {
for key, value := range descriptor.WorkerConfigForm.DefaultValues {
if value == nil {
continue
}
if config.WorkerConfigValues[key] != nil {
continue
}
config.WorkerConfigValues[key] = proto.Clone(value).(*plugin_pb.ConfigValue)
}
}
if descriptor.AdminRuntimeDefaults != nil {
runtime := config.AdminRuntime
defaults := descriptor.AdminRuntimeDefaults
if runtime.DetectionIntervalSeconds <= 0 {
runtime.DetectionIntervalSeconds = defaults.DetectionIntervalSeconds
}
if runtime.DetectionTimeoutSeconds <= 0 {
runtime.DetectionTimeoutSeconds = defaults.DetectionTimeoutSeconds
}
if runtime.MaxJobsPerDetection <= 0 {
runtime.MaxJobsPerDetection = defaults.MaxJobsPerDetection
}
if runtime.GlobalExecutionConcurrency <= 0 {
runtime.GlobalExecutionConcurrency = defaults.GlobalExecutionConcurrency
}
if runtime.PerWorkerExecutionConcurrency <= 0 {
runtime.PerWorkerExecutionConcurrency = defaults.PerWorkerExecutionConcurrency
}
if runtime.RetryBackoffSeconds <= 0 {
runtime.RetryBackoffSeconds = defaults.RetryBackoffSeconds
}
if runtime.RetryLimit < 0 {
runtime.RetryLimit = defaults.RetryLimit
}
}
}
func isBlankStringConfigValue(value *plugin_pb.ConfigValue) bool {
if value == nil {
return true
}
kind, ok := value.Kind.(*plugin_pb.ConfigValue_StringValue)
if !ok {
return false
}
return strings.TrimSpace(kind.StringValue) == ""
}
func parsePositiveInt(raw string, defaultValue int) int {
value, err := strconv.Atoi(strings.TrimSpace(raw))
if err != nil || value <= 0 {
-189
View File
@@ -1,120 +1,11 @@
package dash
import (
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
func TestExpirePluginJobAPI(t *testing.T) {
makeRequest := func(adminServer *AdminServer, jobID string, body io.Reader) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPost, "/api/plugin/jobs/"+jobID+"/expire", body)
req = mux.SetURLVars(req, map[string]string{"jobId": jobID})
recorder := httptest.NewRecorder()
adminServer.ExpirePluginJobAPI(recorder, req)
return recorder
}
t.Run("empty job id", func(t *testing.T) {
recorder := makeRequest(&AdminServer{}, "", nil)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("expected 400, got %d", recorder.Code)
}
})
t.Run("invalid json", func(t *testing.T) {
recorder := makeRequest(&AdminServer{}, "job-id", strings.NewReader("{"))
if recorder.Code != http.StatusBadRequest {
t.Fatalf("expected 400, got %d", recorder.Code)
}
})
t.Run("job not found", func(t *testing.T) {
adminServer := &AdminServer{
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
return nil, false, plugin.ErrJobNotFound
},
}
recorder := makeRequest(adminServer, "missing", strings.NewReader(`{"reason":"nope"}`))
if recorder.Code != http.StatusNotFound {
t.Fatalf("expected 404, got %d", recorder.Code)
}
var payload map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("failed to unmarshal body: %v", err)
}
if payload["error"] == nil {
t.Fatalf("expected error payload, got %v", payload)
}
})
t.Run("successful expire", func(t *testing.T) {
expected := &plugin.TrackedJob{JobID: "foo", State: "assigned"}
adminServer := &AdminServer{
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
if jobID != "foo" {
return nil, false, errors.New("unexpected")
}
return expected, true, nil
},
}
recorder := makeRequest(adminServer, "foo", strings.NewReader(`{"reason":"cleanup"}`))
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Code)
}
var payload map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["job_id"] != "foo" {
t.Fatalf("expected job_id foo, got %v", payload["job_id"])
}
if expired, ok := payload["expired"].(bool); !ok || !expired {
t.Fatalf("expected expired=true, got %v", payload["expired"])
}
jobData, ok := payload["job"].(map[string]any)
if !ok || jobData["job_id"] != "foo" {
t.Fatalf("expected job info with job_id, got %v", payload["job"])
}
})
t.Run("non-active job", func(t *testing.T) {
adminServer := &AdminServer{
expireJobHandler: func(jobID, reason string) (*plugin.TrackedJob, bool, error) {
return nil, false, nil
},
}
recorder := makeRequest(adminServer, "bar", strings.NewReader(`{"reason":"ignore"}`))
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Code)
}
var payload map[string]any
if err := json.Unmarshal(recorder.Body.Bytes(), &payload); err != nil {
t.Fatalf("failed to decode payload: %v", err)
}
if payload["job_id"] != "bar" {
t.Fatalf("expected job_id bar, got %v", payload["job_id"])
}
if expired, ok := payload["expired"].(bool); !ok || expired {
t.Fatalf("expected expired=false, got %v", payload["expired"])
}
if payload["message"] != "job is not active" {
t.Fatalf("expected message job is not active, got %v", payload["message"])
}
if _, exists := payload["job"]; exists {
t.Fatalf("expected no job payload for non-active job, got %v", payload["job"])
}
})
}
func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
t.Parallel()
@@ -140,83 +31,3 @@ func TestBuildJobSpecFromProposalDoesNotReuseProposalID(t *testing.T) {
t.Fatalf("dedupe key must be preserved: got=%s want=%s", jobA.DedupeKey, proposal.DedupeKey)
}
}
func TestApplyDescriptorDefaultsToPersistedConfigBackfillsAdminDefaults(t *testing.T) {
t.Parallel()
config := &plugin_pb.PersistedJobTypeConfig{
JobType: "admin_script",
AdminConfigValues: map[string]*plugin_pb.ConfigValue{},
WorkerConfigValues: map[string]*plugin_pb.ConfigValue{},
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
}
descriptor := &plugin_pb.JobTypeDescriptor{
JobType: "admin_script",
AdminConfigForm: &plugin_pb.ConfigForm{
DefaultValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.balance -apply"},
},
"run_interval_minutes": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 17},
},
},
},
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
DetectionIntervalSeconds: 60,
DetectionTimeoutSeconds: 300,
},
}
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
script := config.AdminConfigValues["script"]
if script == nil {
t.Fatalf("expected script default to be backfilled")
}
scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
if !ok || scriptKind.StringValue == "" {
t.Fatalf("expected non-empty script default, got=%+v", script)
}
if config.AdminRuntime.DetectionIntervalSeconds != 60 {
t.Fatalf("expected runtime detection interval default to be backfilled")
}
}
func TestApplyDescriptorDefaultsToPersistedConfigReplacesBlankAdminScript(t *testing.T) {
t.Parallel()
config := &plugin_pb.PersistedJobTypeConfig{
JobType: "admin_script",
AdminConfigValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: " "},
},
},
AdminRuntime: &plugin_pb.AdminRuntimeConfig{},
}
descriptor := &plugin_pb.JobTypeDescriptor{
JobType: "admin_script",
AdminConfigForm: &plugin_pb.ConfigForm{
DefaultValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: "volume.fix.replication -apply"},
},
},
},
}
applyDescriptorDefaultsToPersistedConfig(config, descriptor)
script := config.AdminConfigValues["script"]
if script == nil {
t.Fatalf("expected script config value")
}
scriptKind, ok := script.Kind.(*plugin_pb.ConfigValue_StringValue)
if !ok {
t.Fatalf("expected string script config value, got=%T", script.Kind)
}
if scriptKind.StringValue != "volume.fix.replication -apply" {
t.Fatalf("expected blank script to be replaced by default, got=%q", scriptKind.StringValue)
}
}
-2
View File
@@ -234,7 +234,6 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
pluginApi.HandleFunc("/jobs/{jobId}/detail", h.adminServer.GetPluginJobDetailAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/activities", h.adminServer.GetPluginActivitiesAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/scheduler-states", h.adminServer.GetPluginSchedulerStatesAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/scheduler-status", h.adminServer.GetPluginSchedulerStatusAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/job-types/{jobType}/descriptor", h.adminServer.GetPluginJobTypeDescriptorAPI).Methods(http.MethodGet)
pluginApi.HandleFunc("/job-types/{jobType}/schema", h.adminServer.RequestPluginJobTypeSchemaAPI).Methods(http.MethodPost)
pluginApi.HandleFunc("/job-types/{jobType}/config", h.adminServer.GetPluginJobTypeConfigAPI).Methods(http.MethodGet)
@@ -243,7 +242,6 @@ func (h *AdminHandlers) registerAPIRoutes(api *mux.Router, enforceWrite bool) {
pluginApi.Handle("/job-types/{jobType}/detect", wrapWrite(h.adminServer.TriggerPluginDetectionAPI)).Methods(http.MethodPost)
pluginApi.Handle("/job-types/{jobType}/run", wrapWrite(h.adminServer.RunPluginJobTypeAPI)).Methods(http.MethodPost)
pluginApi.Handle("/jobs/execute", wrapWrite(h.adminServer.ExecutePluginJobAPI)).Methods(http.MethodPost)
pluginApi.Handle("/jobs/{jobId}/expire", wrapWrite(h.adminServer.ExpirePluginJobAPI)).Methods(http.MethodPost)
mqApi := api.PathPrefix("/mq").Subrouter()
mqApi.HandleFunc("/topics/{namespace}/{topic}", h.mqHandlers.GetTopicDetailsAPI).Methods(http.MethodGet)
@@ -21,9 +21,6 @@ func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_NoAuth(t *testing.T) {
if !hasRoute(router, http.MethodGet, "/api/plugin/jobs/example/detail") {
t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in no-auth mode")
}
if !hasRoute(router, http.MethodPost, "/api/plugin/jobs/example/expire") {
t.Fatalf("expected POST /api/plugin/jobs/:jobId/expire to be registered in no-auth mode")
}
}
func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) {
@@ -37,9 +34,6 @@ func TestSetupRoutes_RegistersPluginSchedulerStatesAPI_WithAuth(t *testing.T) {
if !hasRoute(router, http.MethodGet, "/api/plugin/jobs/example/detail") {
t.Fatalf("expected GET /api/plugin/jobs/:jobId/detail to be registered in auth mode")
}
if !hasRoute(router, http.MethodPost, "/api/plugin/jobs/example/expire") {
t.Fatalf("expected POST /api/plugin/jobs/:jobId/expire to be registered in auth mode")
}
}
func TestSetupRoutes_RegistersPluginPages_NoAuth(t *testing.T) {
-7
View File
@@ -1,7 +0,0 @@
package plugin
// LockManager provides a shared exclusive lock for admin-managed detection/execution.
// Acquire returns a release function that must be called when the protected work finishes.
type LockManager interface {
Acquire(reason string) (release func(), err error)
}
+9 -96
View File
@@ -24,7 +24,6 @@ const (
defaultHeartbeatInterval = 30
defaultReconnectDelay = 5
defaultPendingSchemaBuffer = 1
adminScriptJobType = "admin_script"
)
type Options struct {
@@ -32,9 +31,7 @@ type Options struct {
OutgoingBufferSize int
SendTimeout time.Duration
SchedulerTick time.Duration
IdleSleepDuration time.Duration
ClusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error)
LockManager LockManager
}
// JobTypeInfo contains metadata about a plugin job type.
@@ -54,25 +51,17 @@ type Plugin struct {
sendTimeout time.Duration
schedulerTick time.Duration
idleSleepDuration time.Duration
clusterContextProvider func(context.Context) (*plugin_pb.ClusterContext, error)
lockManager LockManager
schedulerMu sync.Mutex
schedulerPhase string
currentJobType string
iterationStartedAt time.Time
lastIterationEndedAt time.Time
lastIterationWorkDetected bool
schedulerMu sync.Mutex
nextDetectionAt map[string]time.Time
detectionInFlight map[string]bool
detectorLeaseMu sync.Mutex
detectorLeases map[string]string
schedulerExecMu sync.Mutex
schedulerExecReservations map[string]int
adminScriptRunMu sync.RWMutex
schedulerDetectionMu sync.Mutex
schedulerDetection map[string]*schedulerDetectionInfo
dedupeMu sync.Mutex
recentDedupeByType map[string]map[string]time.Time
@@ -91,8 +80,6 @@ type Plugin struct {
jobsMu sync.RWMutex
jobs map[string]*TrackedJob
// serialize stale job cleanup to avoid duplicate expirations
staleJobsMu sync.Mutex
jobDetailsMu sync.Mutex
@@ -151,10 +138,6 @@ func New(options Options) (*Plugin, error) {
if schedulerTick <= 0 {
schedulerTick = defaultSchedulerTick
}
idleSleepDuration := options.IdleSleepDuration
if idleSleepDuration <= 0 {
idleSleepDuration = defaultIdleSleepDuration
}
plugin := &Plugin{
store: store,
@@ -162,17 +145,15 @@ func New(options Options) (*Plugin, error) {
outgoingBuffer: bufferSize,
sendTimeout: sendTimeout,
schedulerTick: schedulerTick,
idleSleepDuration: idleSleepDuration,
clusterContextProvider: options.ClusterContextProvider,
lockManager: options.LockManager,
schedulerPhase: "idle",
sessions: make(map[string]*streamSession),
pendingSchema: make(map[string]chan *plugin_pb.ConfigSchemaResponse),
pendingDetection: make(map[string]*pendingDetectionState),
pendingExecution: make(map[string]chan *plugin_pb.JobCompleted),
nextDetectionAt: make(map[string]time.Time),
detectionInFlight: make(map[string]bool),
detectorLeases: make(map[string]string),
schedulerExecReservations: make(map[string]int),
schedulerDetection: make(map[string]*schedulerDetectionInfo),
recentDedupeByType: make(map[string]map[string]time.Time),
jobs: make(map[string]*TrackedJob),
activities: make([]JobActivity, 0, 256),
@@ -399,13 +380,6 @@ func (r *Plugin) BaseDir() string {
return r.store.BaseDir()
}
func (r *Plugin) acquireAdminLock(reason string) (func(), error) {
if r == nil || r.lockManager == nil {
return func() {}, nil
}
return r.lockManager.Acquire(reason)
}
// RunDetectionWithReport requests one detector worker and returns proposals with request metadata.
func (r *Plugin) RunDetectionWithReport(
ctx context.Context,
@@ -413,9 +387,6 @@ func (r *Plugin) RunDetectionWithReport(
clusterContext *plugin_pb.ClusterContext,
maxResults int32,
) (*DetectionReport, error) {
releaseGate := r.acquireDetectionExecutionGate(jobType, false)
defer releaseGate()
detector, err := r.pickDetector(jobType)
if err != nil {
return nil, err
@@ -430,10 +401,7 @@ func (r *Plugin) RunDetectionWithReport(
if err != nil {
return nil, err
}
lastCompletedRun := r.loadLastSuccessfulRun(jobType)
if strings.EqualFold(strings.TrimSpace(jobType), adminScriptJobType) {
lastCompletedRun = r.loadLastCompletedRun(jobType)
}
lastSuccessfulRun := r.loadLastSuccessfulRun(jobType)
state := &pendingDetectionState{
complete: make(chan *plugin_pb.DetectionComplete, 1),
@@ -474,7 +442,7 @@ func (r *Plugin) RunDetectionWithReport(
AdminConfigValues: adminConfigValues,
WorkerConfigValues: workerConfigValues,
ClusterContext: clusterContext,
LastSuccessfulRun: lastCompletedRun,
LastSuccessfulRun: lastSuccessfulRun,
MaxResults: maxResults,
},
},
@@ -561,14 +529,11 @@ func (r *Plugin) ExecuteJob(
if job == nil {
return nil, fmt.Errorf("job is nil")
}
jobType := strings.TrimSpace(job.JobType)
if jobType == "" {
if strings.TrimSpace(job.JobType) == "" {
return nil, fmt.Errorf("job_type is required")
}
releaseGate := r.acquireDetectionExecutionGate(jobType, true)
defer releaseGate()
executor, err := r.registry.PickExecutor(jobType)
executor, err := r.registry.PickExecutor(job.JobType)
if err != nil {
return nil, err
}
@@ -576,23 +541,6 @@ func (r *Plugin) ExecuteJob(
return r.executeJobWithExecutor(ctx, executor, job, clusterContext, attempt)
}
func (r *Plugin) acquireDetectionExecutionGate(jobType string, execution bool) func() {
normalizedJobType := strings.ToLower(strings.TrimSpace(jobType))
if execution && normalizedJobType == adminScriptJobType {
r.adminScriptRunMu.Lock()
return func() {
r.adminScriptRunMu.Unlock()
}
}
if normalizedJobType != adminScriptJobType {
r.adminScriptRunMu.RLock()
return func() {
r.adminScriptRunMu.RUnlock()
}
}
return func() {}
}
func (r *Plugin) executeJobWithExecutor(
ctx context.Context,
executor *WorkerSession,
@@ -1341,41 +1289,6 @@ func (r *Plugin) loadLastSuccessfulRun(jobType string) *timestamppb.Timestamp {
return timestamppb.New(latest.UTC())
}
func (r *Plugin) loadLastCompletedRun(jobType string) *timestamppb.Timestamp {
history, err := r.store.LoadRunHistory(jobType)
if err != nil {
glog.Warningf("Plugin failed to load run history for %s: %v", jobType, err)
return nil
}
if history == nil {
return nil
}
var latest time.Time
for i := range history.SuccessfulRuns {
completedAt := history.SuccessfulRuns[i].CompletedAt
if completedAt == nil || completedAt.IsZero() {
continue
}
if latest.IsZero() || completedAt.After(latest) {
latest = *completedAt
}
}
for i := range history.ErrorRuns {
completedAt := history.ErrorRuns[i].CompletedAt
if completedAt == nil || completedAt.IsZero() {
continue
}
if latest.IsZero() || completedAt.After(latest) {
latest = *completedAt
}
}
if latest.IsZero() {
return nil
}
return timestamppb.New(latest.UTC())
}
func CloneConfigValueMap(in map[string]*plugin_pb.ConfigValue) map[string]*plugin_pb.ConfigValue {
if len(in) == 0 {
return map[string]*plugin_pb.ConfigValue{}
-164
View File
@@ -4,10 +4,8 @@ import (
"context"
"errors"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestRunDetectionSendsCancelOnContextDone(t *testing.T) {
@@ -112,165 +110,3 @@ func TestExecuteJobSendsCancelOnContextDone(t *testing.T) {
t.Fatalf("expected context canceled error, got %v", runErr)
}
}
func TestAdminScriptExecutionBlocksOtherDetection(t *testing.T) {
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New plugin error: %v", err)
}
defer pluginSvc.Shutdown()
const adminWorkerID = "worker-admin-script"
const otherWorkerID = "worker-vacuum"
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: adminWorkerID,
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "admin_script", CanExecute: true, MaxExecutionConcurrency: 1},
},
})
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: otherWorkerID,
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "vacuum", CanDetect: true, MaxDetectionConcurrency: 1},
},
})
adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
pluginSvc.putSession(adminSession)
pluginSvc.putSession(otherSession)
adminErrCh := make(chan error, 1)
go func() {
_, runErr := pluginSvc.ExecuteJob(context.Background(), &plugin_pb.JobSpec{
JobId: "job-admin-script-1",
JobType: "admin_script",
}, &plugin_pb.ClusterContext{}, 1)
adminErrCh <- runErr
}()
adminExecMessage := <-adminSession.outgoing
if adminExecMessage.GetExecuteJobRequest() == nil {
t.Fatalf("expected admin_script execute request")
}
detectErrCh := make(chan error, 1)
go func() {
_, runErr := pluginSvc.RunDetection(context.Background(), "vacuum", &plugin_pb.ClusterContext{}, 10)
detectErrCh <- runErr
}()
select {
case unexpected := <-otherSession.outgoing:
t.Fatalf("expected vacuum detection to wait while admin_script runs, got message: %+v", unexpected)
case <-time.After(100 * time.Millisecond):
}
pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
RequestId: adminExecMessage.RequestId,
JobId: "job-admin-script-1",
JobType: "admin_script",
Success: true,
CompletedAt: timestamppb.Now(),
})
if runErr := <-adminErrCh; runErr != nil {
t.Fatalf("admin_script ExecuteJob error: %v", runErr)
}
detectMessage := <-otherSession.outgoing
detectRequest := detectMessage.GetRunDetectionRequest()
if detectRequest == nil {
t.Fatalf("expected vacuum detection request after admin_script completion")
}
pluginSvc.handleDetectionComplete(otherWorkerID, &plugin_pb.DetectionComplete{
RequestId: detectMessage.RequestId,
JobType: "vacuum",
Success: true,
})
if runErr := <-detectErrCh; runErr != nil {
t.Fatalf("vacuum RunDetection error: %v", runErr)
}
}
func TestAdminScriptExecutionBlocksOtherExecution(t *testing.T) {
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New plugin error: %v", err)
}
defer pluginSvc.Shutdown()
const adminWorkerID = "worker-admin-script"
const otherWorkerID = "worker-vacuum"
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: adminWorkerID,
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "admin_script", CanExecute: true, MaxExecutionConcurrency: 1},
},
})
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: otherWorkerID,
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "vacuum", CanExecute: true, MaxExecutionConcurrency: 1},
},
})
adminSession := &streamSession{workerID: adminWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
otherSession := &streamSession{workerID: otherWorkerID, outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 8)}
pluginSvc.putSession(adminSession)
pluginSvc.putSession(otherSession)
adminErrCh := make(chan error, 1)
go func() {
_, runErr := pluginSvc.ExecuteJob(context.Background(), &plugin_pb.JobSpec{
JobId: "job-admin-script-2",
JobType: "admin_script",
}, &plugin_pb.ClusterContext{}, 1)
adminErrCh <- runErr
}()
adminExecMessage := <-adminSession.outgoing
if adminExecMessage.GetExecuteJobRequest() == nil {
t.Fatalf("expected admin_script execute request")
}
otherErrCh := make(chan error, 1)
go func() {
_, runErr := pluginSvc.ExecuteJob(context.Background(), &plugin_pb.JobSpec{
JobId: "job-vacuum-1",
JobType: "vacuum",
}, &plugin_pb.ClusterContext{}, 1)
otherErrCh <- runErr
}()
select {
case unexpected := <-otherSession.outgoing:
t.Fatalf("expected vacuum execute to wait while admin_script runs, got message: %+v", unexpected)
case <-time.After(100 * time.Millisecond):
}
pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
RequestId: adminExecMessage.RequestId,
JobId: "job-admin-script-2",
JobType: "admin_script",
Success: true,
CompletedAt: timestamppb.Now(),
})
if runErr := <-adminErrCh; runErr != nil {
t.Fatalf("admin_script ExecuteJob error: %v", runErr)
}
otherExecMessage := <-otherSession.outgoing
if otherExecMessage.GetExecuteJobRequest() == nil {
t.Fatalf("expected vacuum execute request after admin_script completion")
}
pluginSvc.handleJobCompleted(&plugin_pb.JobCompleted{
RequestId: otherExecMessage.RequestId,
JobId: "job-vacuum-1",
JobType: "vacuum",
Success: true,
CompletedAt: timestamppb.Now(),
})
if runErr := <-otherErrCh; runErr != nil {
t.Fatalf("vacuum ExecuteJob error: %v", runErr)
}
}
@@ -195,64 +195,3 @@ func TestRunDetectionWithReportCapturesDetectionActivities(t *testing.T) {
t.Fatalf("expected requested/proposal/completed activities, got stages=%v", stages)
}
}
func TestRunDetectionAdminScriptUsesLastCompletedRun(t *testing.T) {
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New plugin error: %v", err)
}
defer pluginSvc.Shutdown()
jobType := "admin_script"
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: "worker-admin-script",
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: jobType, CanDetect: true, MaxDetectionConcurrency: 1},
},
})
session := &streamSession{workerID: "worker-admin-script", outgoing: make(chan *plugin_pb.AdminToWorkerMessage, 1)}
pluginSvc.putSession(session)
successCompleted := time.Date(2026, 2, 1, 10, 0, 0, 0, time.UTC)
errorCompleted := successCompleted.Add(45 * time.Minute)
if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
Outcome: RunOutcomeSuccess,
CompletedAt: timeToPtr(successCompleted),
}); err != nil {
t.Fatalf("AppendRunRecord success run: %v", err)
}
if err := pluginSvc.store.AppendRunRecord(jobType, &JobRunRecord{
Outcome: RunOutcomeError,
CompletedAt: timeToPtr(errorCompleted),
}); err != nil {
t.Fatalf("AppendRunRecord error run: %v", err)
}
errCh := make(chan error, 1)
go func() {
_, runErr := pluginSvc.RunDetection(context.Background(), jobType, &plugin_pb.ClusterContext{}, 10)
errCh <- runErr
}()
message := <-session.outgoing
detectRequest := message.GetRunDetectionRequest()
if detectRequest == nil {
t.Fatalf("expected run detection request message")
}
if detectRequest.LastSuccessfulRun == nil {
t.Fatalf("expected last_successful_run to be set")
}
if got := detectRequest.LastSuccessfulRun.AsTime().UTC(); !got.Equal(errorCompleted) {
t.Fatalf("unexpected last_successful_run, got=%s want=%s", got, errorCompleted)
}
pluginSvc.handleDetectionComplete("worker-admin-script", &plugin_pb.DetectionComplete{
RequestId: message.RequestId,
JobType: jobType,
Success: true,
})
if runErr := <-errCh; runErr != nil {
t.Fatalf("RunDetection error: %v", runErr)
}
}
-281
View File
@@ -2,7 +2,6 @@ package plugin
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
@@ -10,18 +9,12 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/timestamppb"
)
const (
maxTrackedJobsTotal = 1000
maxActivityRecords = 4000
maxRelatedJobs = 100
// stale active jobs block dedupe and scheduling; use generous defaults to
// avoid expiring legitimate long-running tasks.
defaultStaleActiveJobTimeout = 24 * time.Hour
defaultOrphanedActiveJobTimeout = 15 * time.Minute
)
var (
@@ -30,14 +23,6 @@ var (
StateCanceled = strings.ToLower(plugin_pb.JobState_JOB_STATE_CANCELED.String())
)
type activeJobSnapshot struct {
jobID string
jobType string
workerID string
requestID string
lastUpdate time.Time
}
// activityLess reports whether activity a occurred after activity b (newest-first order).
// A nil OccurredAt is treated as the zero time.
func activityLess(a, b JobActivity) bool {
@@ -69,13 +54,6 @@ func (r *Plugin) loadPersistedMonitorState() error {
if strings.TrimSpace(job.JobID) == "" {
continue
}
if isActiveTrackedJobState(job.State) {
if detail, detailErr := r.store.LoadJobDetail(job.JobID); detailErr != nil {
glog.Warningf("Plugin failed to load detail snapshot for job %s: %v", job.JobID, detailErr)
} else if detail != nil {
mergeTerminalDetailIntoTracked(&job, detail)
}
}
// Backward compatibility: migrate older inline detail payloads
// out of tracked_jobs.json into dedicated per-job detail files.
if hasTrackedJobRichDetails(job) {
@@ -103,265 +81,6 @@ func (r *Plugin) loadPersistedMonitorState() error {
return nil
}
// ExpireJob marks an active job as failed so it no longer blocks scheduling.
func (r *Plugin) ExpireJob(jobID, reason string) (*TrackedJob, bool, error) {
normalizedJobID := strings.TrimSpace(jobID)
if normalizedJobID == "" {
return nil, false, ErrJobNotFound
}
reason = strings.TrimSpace(reason)
if reason == "" {
reason = "job expired by admin request"
}
var jobType string
var requestID string
active := false
r.jobsMu.RLock()
if tracked := r.jobs[normalizedJobID]; tracked != nil {
jobType = tracked.JobType
requestID = tracked.RequestID
active = isActiveTrackedJobState(tracked.State)
}
r.jobsMu.RUnlock()
if jobType == "" || requestID == "" || !active {
if detail, err := r.store.LoadJobDetail(normalizedJobID); err != nil {
return nil, false, err
} else if detail != nil {
if jobType == "" {
jobType = detail.JobType
}
if requestID == "" {
requestID = detail.RequestID
}
if !active && isActiveTrackedJobState(detail.State) {
active = true
}
}
}
if jobType == "" {
return nil, false, ErrJobNotFound
}
if !active {
current, _ := r.GetTrackedJob(normalizedJobID)
if current == nil {
if detail, err := r.store.LoadJobDetail(normalizedJobID); err == nil && detail != nil {
clone := cloneTrackedJob(*detail)
current = &clone
}
}
return current, false, nil
}
now := time.Now().UTC()
r.handleJobCompleted(&plugin_pb.JobCompleted{
JobId: normalizedJobID,
JobType: jobType,
RequestId: requestID,
Success: false,
ErrorMessage: reason,
CompletedAt: timestamppb.New(now),
})
r.appendActivity(JobActivity{
JobID: normalizedJobID,
JobType: jobType,
RequestID: requestID,
Source: "admin_expire",
Message: reason,
Stage: "expired",
OccurredAt: timeToPtr(now),
})
updated, _ := r.GetTrackedJob(normalizedJobID)
return updated, true, nil
}
// expireStaleJobs marks stale active jobs as failed so they stop blocking new work.
func (r *Plugin) expireStaleJobs(now time.Time) int {
if now.IsZero() {
now = time.Now().UTC()
}
r.staleJobsMu.Lock()
defer r.staleJobsMu.Unlock()
snapshots := r.snapshotActiveJobs()
if len(snapshots) == 0 {
return 0
}
expired := 0
for _, snap := range snapshots {
if snap.lastUpdate.IsZero() {
continue
}
if stale, _, _ := r.evaluateStaleJob(now, snap.workerID, snap.lastUpdate); !stale {
continue
}
reason := r.confirmStaleReason(now, snap.jobID)
if reason == "" {
continue
}
r.handleJobCompleted(&plugin_pb.JobCompleted{
JobId: snap.jobID,
JobType: snap.jobType,
RequestId: snap.requestID,
Success: false,
ErrorMessage: reason,
CompletedAt: timestamppb.New(now),
})
expired++
}
return expired
}
func (r *Plugin) snapshotActiveJobs() []activeJobSnapshot {
r.jobsMu.RLock()
defer r.jobsMu.RUnlock()
if len(r.jobs) == 0 {
return nil
}
out := make([]activeJobSnapshot, 0, len(r.jobs))
for _, job := range r.jobs {
if job == nil {
continue
}
if !isActiveTrackedJobState(job.State) {
continue
}
out = append(out, activeJobSnapshot{
jobID: job.JobID,
jobType: job.JobType,
workerID: job.WorkerID,
requestID: job.RequestID,
lastUpdate: jobLastUpdated(job),
})
}
return out
}
func jobLastUpdated(job *TrackedJob) time.Time {
if job == nil {
return time.Time{}
}
if job.UpdatedAt != nil && !job.UpdatedAt.IsZero() {
return *job.UpdatedAt
}
if job.CreatedAt != nil && !job.CreatedAt.IsZero() {
return *job.CreatedAt
}
return time.Time{}
}
func (r *Plugin) evaluateStaleJob(now time.Time, workerID string, lastUpdate time.Time) (bool, time.Duration, string) {
if lastUpdate.IsZero() {
return false, 0, ""
}
timeout := defaultStaleActiveJobTimeout
reason := fmt.Sprintf("job expired after %s without progress", timeout)
workerID = strings.TrimSpace(workerID)
if workerID == "" {
reason = fmt.Sprintf("job expired after %s without executor assignment", timeout)
} else if !r.isWorkerAvailable(workerID) {
timeout = defaultOrphanedActiveJobTimeout
reason = fmt.Sprintf("job expired after %s without worker heartbeat (worker=%s)", timeout, workerID)
}
if now.Sub(lastUpdate) < timeout {
return false, timeout, reason
}
return true, timeout, reason
}
func (r *Plugin) confirmStaleReason(now time.Time, jobID string) string {
r.jobsMu.RLock()
job := r.jobs[jobID]
if job == nil || !isActiveTrackedJobState(job.State) {
r.jobsMu.RUnlock()
return ""
}
lastUpdate := jobLastUpdated(job)
workerID := job.WorkerID
r.jobsMu.RUnlock()
stale, _, reason := r.evaluateStaleJob(now, workerID, lastUpdate)
if !stale {
return ""
}
return reason
}
func (r *Plugin) isWorkerAvailable(workerID string) bool {
workerID = strings.TrimSpace(workerID)
if workerID == "" {
return false
}
_, ok := r.registry.Get(workerID)
return ok
}
func isTerminalTrackedJobState(state string) bool {
normalized := strings.ToLower(strings.TrimSpace(state))
switch normalized {
case StateSucceeded, StateFailed, StateCanceled:
return true
default:
return false
}
}
func mergeTerminalDetailIntoTracked(tracked *TrackedJob, detail *TrackedJob) {
if tracked == nil || detail == nil {
return
}
if !isTerminalTrackedJobState(detail.State) {
return
}
if !isActiveTrackedJobState(tracked.State) {
return
}
if detail.State != "" {
tracked.State = detail.State
}
if detail.Progress != 0 {
tracked.Progress = detail.Progress
}
if detail.Stage != "" {
tracked.Stage = detail.Stage
}
if detail.Message != "" {
tracked.Message = detail.Message
}
if detail.ErrorMessage != "" {
tracked.ErrorMessage = detail.ErrorMessage
}
if detail.ResultSummary != "" {
tracked.ResultSummary = detail.ResultSummary
}
if detail.CompletedAt != nil && !detail.CompletedAt.IsZero() {
tracked.CompletedAt = detail.CompletedAt
}
if detail.UpdatedAt != nil && !detail.UpdatedAt.IsZero() {
tracked.UpdatedAt = detail.UpdatedAt
}
if tracked.UpdatedAt == nil && tracked.CompletedAt != nil {
tracked.UpdatedAt = tracked.CompletedAt
}
}
func (r *Plugin) ListTrackedJobs(jobType string, state string, limit int) []TrackedJob {
r.jobsMu.RLock()
defer r.jobsMu.RUnlock()
+179 -256
View File
@@ -17,8 +17,7 @@ var errExecutorAtCapacity = errors.New("executor is at capacity")
const (
defaultSchedulerTick = 5 * time.Second
defaultIdleSleepDuration = 17 * time.Minute
defaultMaxJobTypeDuration = 30 * time.Minute
defaultScheduledDetectionInterval = 300 * time.Second
defaultScheduledDetectionTimeout = 45 * time.Second
defaultScheduledExecutionTimeout = 90 * time.Second
defaultScheduledMaxResults int32 = 1000
@@ -32,9 +31,9 @@ const (
)
type schedulerPolicy struct {
DetectionInterval time.Duration
DetectionTimeout time.Duration
ExecutionTimeout time.Duration
MaxJobTypeDuration time.Duration
RetryBackoff time.Duration
MaxResults int32
ExecutionConcurrency int
@@ -43,63 +42,31 @@ type schedulerPolicy struct {
ExecutorReserveBackoff time.Duration
}
func (r *Plugin) setSchedulerPhase(phase, jobType string) {
r.schedulerMu.Lock()
r.schedulerPhase = phase
r.currentJobType = jobType
r.schedulerMu.Unlock()
}
func (r *Plugin) schedulerLoop() {
defer r.wg.Done()
ticker := time.NewTicker(r.schedulerTick)
defer ticker.Stop()
// Try once immediately on startup.
workDetected := r.runSchedulerIteration()
r.runSchedulerTick()
for {
if workDetected {
// Immediate re-iteration when work was found.
select {
case <-r.shutdownCh:
return
default:
}
} else {
r.setSchedulerPhase("idle", "")
if !waitForShutdownOrTimer(r.shutdownCh, r.idleSleepDuration) {
return
}
}
select {
case <-r.shutdownCh:
return
default:
case <-ticker.C:
r.runSchedulerTick()
}
workDetected = r.runSchedulerIteration()
}
}
func (r *Plugin) runSchedulerIteration() bool {
now := time.Now().UTC()
r.schedulerMu.Lock()
r.iterationStartedAt = now
r.schedulerMu.Unlock()
r.expireStaleJobs(now)
func (r *Plugin) runSchedulerTick() {
jobTypes := r.registry.DetectableJobTypes()
if len(jobTypes) == 0 {
r.finishIteration(false)
return false
return
}
active := make(map[string]struct{}, len(jobTypes))
enabledJobTypes := make([]string, 0, len(jobTypes))
policies := make(map[string]schedulerPolicy, len(jobTypes))
for _, jobType := range jobTypes {
active[jobType] = struct{}{}
@@ -113,171 +80,19 @@ func (r *Plugin) runSchedulerIteration() bool {
continue
}
enabledJobTypes = append(enabledJobTypes, jobType)
policies[jobType] = policy
if !r.markDetectionDue(jobType, policy.DetectionInterval) {
continue
}
r.wg.Add(1)
go func(jt string, p schedulerPolicy) {
defer r.wg.Done()
r.runScheduledDetection(jt, p)
}(jobType, policy)
}
r.pruneSchedulerState(active)
r.pruneDetectorLeases(active)
if len(enabledJobTypes) == 0 {
r.finishIteration(false)
return false
}
// Acquire the lock ONCE for the entire iteration.
r.setSchedulerPhase("acquiring_lock", "")
releaseLock, lockErr := r.acquireAdminLock("plugin scheduler iteration")
if lockErr != nil {
glog.Warningf("Plugin scheduler failed to acquire lock: %v", lockErr)
r.appendActivity(JobActivity{
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduler iteration aborted: failed to acquire lock: %v", lockErr),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
r.finishIteration(false)
return false
}
defer releaseLock()
// Load cluster context ONCE for all job types.
clusterContext, err := r.loadSchedulerClusterContext()
if err != nil {
r.appendActivity(JobActivity{
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduler iteration aborted: %v", err),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
r.finishIteration(false)
return false
}
// Process each job type sequentially.
anyWorkDetected := false
for _, jobType := range enabledJobTypes {
select {
case <-r.shutdownCh:
r.finishIteration(anyWorkDetected)
return anyWorkDetected
default:
}
policy := policies[jobType]
r.setSchedulerPhase("processing", jobType)
if r.runJobTypeIteration(jobType, policy, clusterContext) {
anyWorkDetected = true
}
}
r.finishIteration(anyWorkDetected)
return anyWorkDetected
}
func (r *Plugin) finishIteration(workDetected bool) {
now := time.Now().UTC()
r.schedulerMu.Lock()
r.lastIterationEndedAt = now
r.lastIterationWorkDetected = workDetected
r.currentJobType = ""
r.schedulerPhase = "idle"
r.schedulerMu.Unlock()
}
func (r *Plugin) runJobTypeIteration(
jobType string,
policy schedulerPolicy,
clusterContext *plugin_pb.ClusterContext,
) bool {
budget := policy.MaxJobTypeDuration
if budget <= 0 {
budget = defaultMaxJobTypeDuration
}
ctx, cancel := context.WithTimeout(r.ctx, budget)
defer cancel()
start := time.Now().UTC()
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: "scheduled detection started",
Stage: "detecting",
OccurredAt: timeToPtr(start),
})
if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip {
r.recordSchedulerDetectionSkip(jobType, fmt.Sprintf("waiting backlog %d reached threshold %d", waitingCount, waitingThreshold))
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection skipped: waiting backlog %d reached threshold %d", waitingCount, waitingThreshold),
Stage: "skipped_waiting_backlog",
OccurredAt: timeToPtr(time.Now().UTC()),
})
return false
}
detectionTimeout := policy.DetectionTimeout
if detectionTimeout <= 0 {
detectionTimeout = defaultScheduledDetectionTimeout
}
detCtx, detCancel := context.WithTimeout(ctx, detectionTimeout)
proposals, err := r.RunDetection(detCtx, jobType, clusterContext, policy.MaxResults)
detCancel()
if err != nil {
r.recordSchedulerDetectionError(jobType, err)
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection failed: %v", err),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
return false
}
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection completed: %d proposal(s)", len(proposals)),
Stage: "detected",
OccurredAt: timeToPtr(time.Now().UTC()),
})
r.recordSchedulerDetectionSuccess(jobType, len(proposals))
filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
if skippedActive > 0 {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection skipped %d proposal(s) due to active assigned/running jobs", skippedActive),
Stage: "deduped_active_jobs",
OccurredAt: timeToPtr(time.Now().UTC()),
})
}
if len(filteredByActive) == 0 {
return false
}
filtered := r.filterScheduledProposals(filteredByActive)
if len(filtered) != len(filteredByActive) {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection deduped %d proposal(s) within this run", len(filteredByActive)-len(filtered)),
Stage: "deduped",
OccurredAt: timeToPtr(time.Now().UTC()),
})
}
if len(filtered) == 0 {
return false
}
r.dispatchScheduledProposals(ctx, jobType, filtered, clusterContext, policy)
return true
}
func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, error) {
@@ -299,9 +114,9 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
}
policy := schedulerPolicy{
DetectionInterval: durationFromSeconds(adminRuntime.DetectionIntervalSeconds, defaultScheduledDetectionInterval),
DetectionTimeout: durationFromSeconds(adminRuntime.DetectionTimeoutSeconds, defaultScheduledDetectionTimeout),
ExecutionTimeout: defaultScheduledExecutionTimeout,
MaxJobTypeDuration: defaultMaxJobTypeDuration,
RetryBackoff: durationFromSeconds(adminRuntime.RetryBackoffSeconds, defaultScheduledRetryBackoff),
MaxResults: adminRuntime.MaxJobsPerDetection,
ExecutionConcurrency: int(adminRuntime.GlobalExecutionConcurrency),
@@ -310,6 +125,9 @@ func (r *Plugin) loadSchedulerPolicy(jobType string) (schedulerPolicy, bool, err
ExecutorReserveBackoff: 200 * time.Millisecond,
}
if policy.DetectionInterval < r.schedulerTick {
policy.DetectionInterval = r.schedulerTick
}
if policy.MaxResults <= 0 {
policy.MaxResults = defaultScheduledMaxResults
}
@@ -346,18 +164,28 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
}
r.schedulerMu.Lock()
currentJobType := r.currentJobType
nextDetectionAt := make(map[string]time.Time, len(r.nextDetectionAt))
for jobType, nextRun := range r.nextDetectionAt {
nextDetectionAt[jobType] = nextRun
}
detectionInFlight := make(map[string]bool, len(r.detectionInFlight))
for jobType, inFlight := range r.detectionInFlight {
detectionInFlight[jobType] = inFlight
}
r.schedulerMu.Unlock()
states := make([]SchedulerJobTypeState, 0, len(jobTypes))
for _, jobTypeInfo := range jobTypes {
jobType := jobTypeInfo.JobType
state := SchedulerJobTypeState{
JobType: jobType,
JobType: jobType,
DetectionInFlight: detectionInFlight[jobType],
}
// Mark as in-flight if this job type is being processed right now.
state.DetectionInFlight = jobType == currentJobType && currentJobType != ""
if nextRun, ok := nextDetectionAt[jobType]; ok && !nextRun.IsZero() {
nextRunUTC := nextRun.UTC()
state.NextDetectionAt = &nextRunUTC
}
policy, enabled, loadErr := r.loadSchedulerPolicy(jobType)
@@ -366,9 +194,9 @@ func (r *Plugin) ListSchedulerStates() ([]SchedulerJobTypeState, error) {
} else {
state.Enabled = enabled
if enabled {
state.DetectionIntervalSeconds = secondsFromDuration(policy.DetectionInterval)
state.DetectionTimeoutSeconds = secondsFromDuration(policy.DetectionTimeout)
state.ExecutionTimeoutSeconds = secondsFromDuration(policy.ExecutionTimeout)
state.MaxJobTypeDurationSeconds = secondsFromDuration(policy.MaxJobTypeDuration)
state.MaxJobsPerDetection = policy.MaxResults
state.GlobalExecutionConcurrency = policy.ExecutionConcurrency
state.PerWorkerExecutionConcurrency = policy.PerWorkerConcurrency
@@ -431,7 +259,49 @@ func deriveSchedulerAdminRuntime(
}
}
func (r *Plugin) markDetectionDue(jobType string, interval time.Duration) bool {
now := time.Now().UTC()
r.schedulerMu.Lock()
defer r.schedulerMu.Unlock()
if r.detectionInFlight[jobType] {
return false
}
nextRun, exists := r.nextDetectionAt[jobType]
if exists && now.Before(nextRun) {
return false
}
r.nextDetectionAt[jobType] = now.Add(interval)
r.detectionInFlight[jobType] = true
return true
}
func (r *Plugin) finishDetection(jobType string) {
r.schedulerMu.Lock()
delete(r.detectionInFlight, jobType)
r.schedulerMu.Unlock()
}
func (r *Plugin) pruneSchedulerState(activeJobTypes map[string]struct{}) {
r.schedulerMu.Lock()
defer r.schedulerMu.Unlock()
for jobType := range r.nextDetectionAt {
if _, ok := activeJobTypes[jobType]; !ok {
delete(r.nextDetectionAt, jobType)
delete(r.detectionInFlight, jobType)
}
}
}
func (r *Plugin) clearSchedulerJobType(jobType string) {
r.schedulerMu.Lock()
delete(r.nextDetectionAt, jobType)
delete(r.detectionInFlight, jobType)
r.schedulerMu.Unlock()
r.clearDetectorLease(jobType, "")
}
@@ -446,6 +316,96 @@ func (r *Plugin) pruneDetectorLeases(activeJobTypes map[string]struct{}) {
}
}
func (r *Plugin) runScheduledDetection(jobType string, policy schedulerPolicy) {
defer r.finishDetection(jobType)
start := time.Now().UTC()
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: "scheduled detection started",
Stage: "detecting",
OccurredAt: timeToPtr(start),
})
if skip, waitingCount, waitingThreshold := r.shouldSkipDetectionForWaitingJobs(jobType, policy); skip {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection skipped: waiting backlog %d reached threshold %d", waitingCount, waitingThreshold),
Stage: "skipped_waiting_backlog",
OccurredAt: timeToPtr(time.Now().UTC()),
})
return
}
clusterContext, err := r.loadSchedulerClusterContext()
if err != nil {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection aborted: %v", err),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
return
}
ctx, cancel := context.WithTimeout(context.Background(), policy.DetectionTimeout)
proposals, err := r.RunDetection(ctx, jobType, clusterContext, policy.MaxResults)
cancel()
if err != nil {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection failed: %v", err),
Stage: "failed",
OccurredAt: timeToPtr(time.Now().UTC()),
})
return
}
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection completed: %d proposal(s)", len(proposals)),
Stage: "detected",
OccurredAt: timeToPtr(time.Now().UTC()),
})
filteredByActive, skippedActive := r.filterProposalsWithActiveJobs(jobType, proposals)
if skippedActive > 0 {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection skipped %d proposal(s) due to active assigned/running jobs", skippedActive),
Stage: "deduped_active_jobs",
OccurredAt: timeToPtr(time.Now().UTC()),
})
}
if len(filteredByActive) == 0 {
return
}
filtered := r.filterScheduledProposals(filteredByActive)
if len(filtered) != len(filteredByActive) {
r.appendActivity(JobActivity{
JobType: jobType,
Source: "admin_scheduler",
Message: fmt.Sprintf("scheduled detection deduped %d proposal(s) within this run", len(filteredByActive)-len(filtered)),
Stage: "deduped",
OccurredAt: timeToPtr(time.Now().UTC()),
})
}
if len(filtered) == 0 {
return
}
r.dispatchScheduledProposals(jobType, filtered, clusterContext, policy)
}
func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error) {
if r.clusterContextProvider == nil {
return nil, fmt.Errorf("cluster context provider is not configured")
@@ -465,7 +425,6 @@ func (r *Plugin) loadSchedulerClusterContext() (*plugin_pb.ClusterContext, error
}
func (r *Plugin) dispatchScheduledProposals(
parentCtx context.Context,
jobType string,
proposals []*plugin_pb.JobProposal,
clusterContext *plugin_pb.ClusterContext,
@@ -479,9 +438,6 @@ func (r *Plugin) dispatchScheduledProposals(
case <-r.shutdownCh:
close(jobQueue)
return
case <-parentCtx.Done():
close(jobQueue)
return
default:
jobQueue <- job
}
@@ -507,8 +463,6 @@ func (r *Plugin) dispatchScheduledProposals(
select {
case <-r.shutdownCh:
return
case <-parentCtx.Done():
return
default:
}
@@ -516,12 +470,10 @@ func (r *Plugin) dispatchScheduledProposals(
select {
case <-r.shutdownCh:
return
case <-parentCtx.Done():
return
default:
}
executor, release, reserveErr := r.reserveScheduledExecutor(parentCtx, jobType, policy)
executor, release, reserveErr := r.reserveScheduledExecutor(jobType, policy)
if reserveErr != nil {
select {
case <-r.shutdownCh:
@@ -541,7 +493,7 @@ func (r *Plugin) dispatchScheduledProposals(
break
}
err := r.executeScheduledJobWithExecutor(parentCtx, executor, job, clusterContext, policy)
err := r.executeScheduledJobWithExecutor(executor, job, clusterContext, policy)
release()
if errors.Is(err, errExecutorAtCapacity) {
r.trackExecutionQueued(job)
@@ -586,7 +538,6 @@ func (r *Plugin) dispatchScheduledProposals(
}
func (r *Plugin) reserveScheduledExecutor(
ctx context.Context,
jobType string,
policy schedulerPolicy,
) (*WorkerSession, func(), error) {
@@ -599,8 +550,6 @@ func (r *Plugin) reserveScheduledExecutor(
select {
case <-r.shutdownCh:
return nil, nil, fmt.Errorf("plugin is shutting down")
case <-ctx.Done():
return nil, nil, ctx.Err()
default:
}
@@ -610,8 +559,8 @@ func (r *Plugin) reserveScheduledExecutor(
executors, err := r.registry.ListExecutors(jobType)
if err != nil {
if !waitForShutdownOrCtx(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
return nil, nil, fmt.Errorf("plugin is shutting down or context canceled")
if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
return nil, nil, fmt.Errorf("plugin is shutting down")
}
continue
}
@@ -624,8 +573,8 @@ func (r *Plugin) reserveScheduledExecutor(
return executor, release, nil
}
if !waitForShutdownOrCtx(r.shutdownCh, ctx, policy.ExecutorReserveBackoff) {
return nil, nil, fmt.Errorf("plugin is shutting down or context canceled")
if !waitForShutdownOrTimer(r.shutdownCh, policy.ExecutorReserveBackoff) {
return nil, nil, fmt.Errorf("plugin is shutting down")
}
}
}
@@ -709,7 +658,6 @@ func schedulerWorkerExecutionLimit(executor *WorkerSession, jobType string, poli
}
func (r *Plugin) executeScheduledJobWithExecutor(
parentCtx context.Context,
executor *WorkerSession,
job *plugin_pb.JobSpec,
clusterContext *plugin_pb.ClusterContext,
@@ -725,12 +673,10 @@ func (r *Plugin) executeScheduledJobWithExecutor(
select {
case <-r.shutdownCh:
return fmt.Errorf("plugin is shutting down")
case <-parentCtx.Done():
return parentCtx.Err()
default:
}
execCtx, cancel := context.WithTimeout(parentCtx, policy.ExecutionTimeout)
execCtx, cancel := context.WithTimeout(context.Background(), policy.ExecutionTimeout)
_, err := r.executeJobWithExecutor(execCtx, executor, job, clusterContext, int32(attempt))
cancel()
if err == nil {
@@ -750,8 +696,8 @@ func (r *Plugin) executeScheduledJobWithExecutor(
Stage: "retry",
OccurredAt: timeToPtr(time.Now().UTC()),
})
if !waitForShutdownOrCtx(r.shutdownCh, parentCtx, policy.RetryBackoff) {
return fmt.Errorf("plugin is shutting down or context canceled")
if !waitForShutdownOrTimer(r.shutdownCh, policy.RetryBackoff) {
return fmt.Errorf("plugin is shutting down")
}
}
}
@@ -893,34 +839,11 @@ func waitForShutdownOrTimer(shutdown <-chan struct{}, duration time.Duration) bo
}
}
func waitForShutdownOrCtx(shutdown <-chan struct{}, ctx context.Context, duration time.Duration) bool {
if duration <= 0 {
return true
}
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-shutdown:
return false
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
// filterProposalsWithActiveJobs removes proposals whose dedupe keys already have active jobs.
// It first expires stale tracked jobs via expireStaleJobs, which can mutate scheduler state,
// so callers should treat this method as a stateful operation.
func (r *Plugin) filterProposalsWithActiveJobs(jobType string, proposals []*plugin_pb.JobProposal) ([]*plugin_pb.JobProposal, int) {
if len(proposals) == 0 {
return proposals, 0
}
r.expireStaleJobs(time.Now().UTC())
activeKeys := make(map[string]struct{})
r.jobsMu.RLock()
for _, job := range r.jobs {
+18 -345
View File
@@ -1,9 +1,7 @@
package plugin
import (
"context"
"fmt"
"sync"
"testing"
"time"
@@ -55,9 +53,6 @@ func TestLoadSchedulerPolicyUsesAdminConfig(t *testing.T) {
if policy.RetryLimit != 4 {
t.Fatalf("unexpected retry limit: got=%d", policy.RetryLimit)
}
if policy.MaxJobTypeDuration != defaultMaxJobTypeDuration {
t.Fatalf("unexpected max job type duration: got=%v", policy.MaxJobTypeDuration)
}
}
func TestLoadSchedulerPolicyUsesDescriptorDefaultsWhenConfigMissing(t *testing.T) {
@@ -131,13 +126,13 @@ func TestReserveScheduledExecutorRespectsPerWorkerLimit(t *testing.T) {
ExecutorReserveBackoff: time.Millisecond,
}
executor1, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
executor1, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy)
if err != nil {
t.Fatalf("reserve executor 1: %v", err)
}
defer release1()
executor2, release2, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
executor2, release2, err := pluginSvc.reserveScheduledExecutor("balance", policy)
if err != nil {
t.Fatalf("reserve executor 2: %v", err)
}
@@ -259,7 +254,7 @@ func TestReserveScheduledExecutorTimesOutWhenNoExecutor(t *testing.T) {
start := time.Now()
pluginSvc.Shutdown()
_, _, err = pluginSvc.reserveScheduledExecutor(context.Background(), "missing-job-type", policy)
_, _, err = pluginSvc.reserveScheduledExecutor("missing-job-type", policy)
if err == nil {
t.Fatalf("expected reservation shutdown error")
}
@@ -290,7 +285,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
ExecutorReserveBackoff: 5 * time.Millisecond,
}
_, release1, err := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
_, release1, err := pluginSvc.reserveScheduledExecutor("balance", policy)
if err != nil {
t.Fatalf("reserve executor 1: %v", err)
}
@@ -301,7 +296,7 @@ func TestReserveScheduledExecutorWaitsForWorkerCapacity(t *testing.T) {
}
secondReserveCh := make(chan reserveResult, 1)
go func() {
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor(context.Background(), "balance", policy)
_, release2, reserveErr := pluginSvc.reserveScheduledExecutor("balance", policy)
if release2 != nil {
release2()
}
@@ -412,9 +407,10 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
},
})
// Mark this job type as currently processing to test DetectionInFlight.
nextDetectionAt := time.Now().UTC().Add(2 * time.Minute).Round(time.Second)
pluginSvc.schedulerMu.Lock()
pluginSvc.currentJobType = jobType
pluginSvc.nextDetectionAt[jobType] = nextDetectionAt
pluginSvc.detectionInFlight[jobType] = true
pluginSvc.schedulerMu.Unlock()
states, err := pluginSvc.ListSchedulerStates()
@@ -433,7 +429,16 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
t.Fatalf("unexpected policy error: %s", state.PolicyError)
}
if !state.DetectionInFlight {
t.Fatalf("expected detection in flight when current job type matches")
t.Fatalf("expected detection in flight")
}
if state.NextDetectionAt == nil {
t.Fatalf("expected next detection time")
}
if state.NextDetectionAt.Unix() != nextDetectionAt.Unix() {
t.Fatalf("unexpected next detection time: got=%v want=%v", state.NextDetectionAt, nextDetectionAt)
}
if state.DetectionIntervalSeconds != 45 {
t.Fatalf("unexpected detection interval: got=%d", state.DetectionIntervalSeconds)
}
if state.DetectionTimeoutSeconds != 30 {
t.Fatalf("unexpected detection timeout: got=%d", state.DetectionTimeoutSeconds)
@@ -441,9 +446,6 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
if state.ExecutionTimeoutSeconds != 90 {
t.Fatalf("unexpected execution timeout: got=%d", state.ExecutionTimeoutSeconds)
}
if state.MaxJobTypeDurationSeconds != int32(defaultMaxJobTypeDuration/time.Second) {
t.Fatalf("unexpected max job type duration: got=%d", state.MaxJobTypeDurationSeconds)
}
if state.MaxJobsPerDetection != 80 {
t.Fatalf("unexpected max jobs per detection: got=%d", state.MaxJobsPerDetection)
}
@@ -465,23 +467,6 @@ func TestListSchedulerStatesIncludesPolicyAndState(t *testing.T) {
if state.ExecutorWorkerCount != 1 {
t.Fatalf("unexpected executor worker count: got=%d", state.ExecutorWorkerCount)
}
// Clear the current job type and verify DetectionInFlight is false.
pluginSvc.schedulerMu.Lock()
pluginSvc.currentJobType = ""
pluginSvc.schedulerMu.Unlock()
states2, err := pluginSvc.ListSchedulerStates()
if err != nil {
t.Fatalf("ListSchedulerStates (2): %v", err)
}
state2 := findSchedulerState(states2, jobType)
if state2 == nil {
t.Fatalf("missing scheduler state for %s (2)", jobType)
}
if state2.DetectionInFlight {
t.Fatalf("expected detection not in flight when current job type is empty")
}
}
func TestListSchedulerStatesShowsDisabledWhenNoPolicy(t *testing.T) {
@@ -596,315 +581,3 @@ func TestPickDetectorReassignsWhenLeaseIsStale(t *testing.T) {
t.Fatalf("expected detector lease to be updated to worker-a, got=%s", lease)
}
}
// mockLockManager records lock/release calls for testing.
type mockLockManager struct {
mu sync.Mutex
acquireCount int
releaseCount int
failAcquire bool
}
func (m *mockLockManager) Acquire(reason string) (func(), error) {
m.mu.Lock()
defer m.mu.Unlock()
if m.failAcquire {
return nil, fmt.Errorf("mock lock acquisition failed")
}
m.acquireCount++
return func() {
m.mu.Lock()
m.releaseCount++
m.mu.Unlock()
}, nil
}
func (m *mockLockManager) Status() interface{} {
return nil
}
func (m *mockLockManager) getAcquireCount() int {
m.mu.Lock()
defer m.mu.Unlock()
return m.acquireCount
}
func (m *mockLockManager) getReleaseCount() int {
m.mu.Lock()
defer m.mu.Unlock()
return m.releaseCount
}
func TestIterationAcquiresLockOnce(t *testing.T) {
t.Parallel()
lock := &mockLockManager{}
pluginSvc, err := New(Options{
LockManager: lock,
})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
// Register two enabled job types.
for _, jt := range []string{"vacuum", "balance"} {
err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
JobType: jt,
AdminRuntime: &plugin_pb.AdminRuntimeConfig{
Enabled: true,
DetectionTimeoutSeconds: 5,
MaxJobsPerDetection: 10,
GlobalExecutionConcurrency: 1,
PerWorkerExecutionConcurrency: 1,
},
})
if err != nil {
t.Fatalf("SaveJobTypeConfig(%s): %v", jt, err)
}
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: "worker-" + jt,
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: jt, CanDetect: true, CanExecute: true},
},
})
}
// runSchedulerIteration requires a cluster context provider.
pluginSvc.clusterContextProvider = func(_ context.Context) (*plugin_pb.ClusterContext, error) {
return &plugin_pb.ClusterContext{}, nil
}
pluginSvc.runSchedulerIteration()
// Lock should have been acquired exactly once (not per-job-type).
if lock.getAcquireCount() != 1 {
t.Fatalf("expected 1 lock acquisition, got %d", lock.getAcquireCount())
}
if lock.getReleaseCount() != 1 {
t.Fatalf("expected 1 lock release, got %d", lock.getReleaseCount())
}
}
func TestIterationReturnsfalseWhenLockFails(t *testing.T) {
t.Parallel()
lock := &mockLockManager{failAcquire: true}
pluginSvc, err := New(Options{
LockManager: lock,
})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
JobType: "vacuum",
AdminRuntime: &plugin_pb.AdminRuntimeConfig{
Enabled: true,
DetectionTimeoutSeconds: 5,
MaxJobsPerDetection: 10,
GlobalExecutionConcurrency: 1,
PerWorkerExecutionConcurrency: 1,
},
})
if err != nil {
t.Fatalf("SaveJobTypeConfig: %v", err)
}
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: "worker-a",
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "vacuum", CanDetect: true, CanExecute: true},
},
})
pluginSvc.clusterContextProvider = func(_ context.Context) (*plugin_pb.ClusterContext, error) {
return &plugin_pb.ClusterContext{}, nil
}
result := pluginSvc.runSchedulerIteration()
if result {
t.Fatalf("expected false when lock acquisition fails")
}
}
func TestSchedulerPhaseTracking(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
// Initial phase should be idle.
pluginSvc.schedulerMu.Lock()
phase := pluginSvc.schedulerPhase
pluginSvc.schedulerMu.Unlock()
if phase != "idle" {
t.Fatalf("expected initial phase to be idle, got=%s", phase)
}
pluginSvc.setSchedulerPhase("processing", "vacuum")
pluginSvc.schedulerMu.Lock()
phase = pluginSvc.schedulerPhase
jobType := pluginSvc.currentJobType
pluginSvc.schedulerMu.Unlock()
if phase != "processing" {
t.Fatalf("expected phase processing, got=%s", phase)
}
if jobType != "vacuum" {
t.Fatalf("expected current job type vacuum, got=%s", jobType)
}
pluginSvc.finishIteration(true)
pluginSvc.schedulerMu.Lock()
phase = pluginSvc.schedulerPhase
jobType = pluginSvc.currentJobType
workDetected := pluginSvc.lastIterationWorkDetected
lastEnded := pluginSvc.lastIterationEndedAt
pluginSvc.schedulerMu.Unlock()
if phase != "idle" {
t.Fatalf("expected phase idle after finish, got=%s", phase)
}
if jobType != "" {
t.Fatalf("expected empty job type after finish, got=%s", jobType)
}
if !workDetected {
t.Fatalf("expected last iteration work detected to be true")
}
if lastEnded.IsZero() {
t.Fatalf("expected last iteration ended at to be set")
}
}
func TestGetSchedulerStatusIncludesIterationFields(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{
IdleSleepDuration: 10 * time.Minute,
})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
now := time.Now().UTC()
pluginSvc.schedulerMu.Lock()
pluginSvc.schedulerPhase = "processing"
pluginSvc.currentJobType = "vacuum"
pluginSvc.iterationStartedAt = now.Add(-5 * time.Second)
pluginSvc.lastIterationEndedAt = now.Add(-20 * time.Second)
pluginSvc.lastIterationWorkDetected = true
pluginSvc.schedulerMu.Unlock()
status := pluginSvc.GetSchedulerStatus()
if status.Phase != "processing" {
t.Fatalf("expected phase processing, got=%s", status.Phase)
}
if status.CurrentJobType != "vacuum" {
t.Fatalf("expected current job type vacuum, got=%s", status.CurrentJobType)
}
if status.IdleSleepSeconds != 600 {
t.Fatalf("expected idle sleep 600s, got=%d", status.IdleSleepSeconds)
}
if status.IterationStartedAt == nil {
t.Fatalf("expected iteration started at to be set")
}
if status.LastIterationEndedAt == nil {
t.Fatalf("expected last iteration ended at to be set")
}
if !status.LastIterationWorkDetected {
t.Fatalf("expected last iteration work detected to be true")
}
// SchedulerTickSeconds should match IdleSleepSeconds for backward compat.
if status.SchedulerTickSeconds != 600 {
t.Fatalf("expected scheduler tick seconds to match idle sleep, got=%d", status.SchedulerTickSeconds)
}
}
func TestGracefulShutdownDuringIteration(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{
IdleSleepDuration: time.Millisecond,
ClusterContextProvider: func(_ context.Context) (*plugin_pb.ClusterContext, error) {
return &plugin_pb.ClusterContext{}, nil
},
})
if err != nil {
t.Fatalf("New: %v", err)
}
// Register an enabled job type so the scheduler loop has work to consider.
err = pluginSvc.SaveJobTypeConfig(&plugin_pb.PersistedJobTypeConfig{
JobType: "vacuum",
AdminRuntime: &plugin_pb.AdminRuntimeConfig{
Enabled: true,
DetectionTimeoutSeconds: 5,
MaxJobsPerDetection: 10,
GlobalExecutionConcurrency: 1,
PerWorkerExecutionConcurrency: 1,
},
})
if err != nil {
t.Fatalf("SaveJobTypeConfig: %v", err)
}
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: "worker-a",
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: "vacuum", CanDetect: true, CanExecute: true},
},
})
// Shutdown while the scheduler loop is actively iterating.
done := make(chan struct{})
go func() {
pluginSvc.Shutdown()
close(done)
}()
select {
case <-done:
// Good — clean shutdown.
case <-time.After(5 * time.Second):
t.Fatalf("shutdown did not complete in time")
}
}
func TestIdleSleepDurationDefault(t *testing.T) {
t.Parallel()
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
if pluginSvc.idleSleepDuration != defaultIdleSleepDuration {
t.Fatalf("expected default idle sleep %v, got=%v", defaultIdleSleepDuration, pluginSvc.idleSleepDuration)
}
}
func TestIdleSleepDurationCustom(t *testing.T) {
t.Parallel()
customDuration := 5 * time.Minute
pluginSvc, err := New(Options{
IdleSleepDuration: customDuration,
})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
if pluginSvc.idleSleepDuration != customDuration {
t.Fatalf("expected custom idle sleep %v, got=%v", customDuration, pluginSvc.idleSleepDuration)
}
}
-257
View File
@@ -1,257 +0,0 @@
package plugin
import (
"sort"
"strings"
"time"
)
type SchedulerStatus struct {
Now time.Time `json:"now"`
SchedulerTickSeconds int `json:"scheduler_tick_seconds"`
IdleSleepSeconds int `json:"idle_sleep_seconds"`
Phase string `json:"phase"`
CurrentJobType string `json:"current_job_type,omitempty"`
IterationStartedAt *time.Time `json:"iteration_started_at,omitempty"`
LastIterationEndedAt *time.Time `json:"last_iteration_ended_at,omitempty"`
LastIterationWorkDetected bool `json:"last_iteration_work_detected"`
Waiting []SchedulerWaitingStatus `json:"waiting,omitempty"`
InProcessJobs []SchedulerJobStatus `json:"in_process_jobs,omitempty"`
JobTypes []SchedulerJobTypeStatus `json:"job_types,omitempty"`
}
type SchedulerWaitingStatus struct {
Reason string `json:"reason"`
JobType string `json:"job_type,omitempty"`
Since *time.Time `json:"since,omitempty"`
Until *time.Time `json:"until,omitempty"`
Details map[string]interface{} `json:"details,omitempty"`
}
type SchedulerJobStatus struct {
JobID string `json:"job_id"`
JobType string `json:"job_type"`
State string `json:"state"`
Stage string `json:"stage,omitempty"`
WorkerID string `json:"worker_id,omitempty"`
Message string `json:"message,omitempty"`
Progress float64 `json:"progress,omitempty"`
CreatedAt *time.Time `json:"created_at,omitempty"`
UpdatedAt *time.Time `json:"updated_at,omitempty"`
DurationSeconds float64 `json:"duration_seconds,omitempty"`
}
type SchedulerJobTypeStatus struct {
JobType string `json:"job_type"`
Enabled bool `json:"enabled"`
DetectionInFlight bool `json:"detection_in_flight"`
MaxJobTypeDurationSeconds int32 `json:"max_job_type_duration_seconds,omitempty"`
LastDetectedAt *time.Time `json:"last_detected_at,omitempty"`
LastDetectedCount int `json:"last_detected_count,omitempty"`
LastDetectionError string `json:"last_detection_error,omitempty"`
LastDetectionSkipped string `json:"last_detection_skipped,omitempty"`
}
type schedulerDetectionInfo struct {
lastDetectedAt time.Time
lastDetectedCount int
lastErrorAt time.Time
lastError string
lastSkippedAt time.Time
lastSkippedReason string
}
func (r *Plugin) recordSchedulerDetectionSuccess(jobType string, count int) {
if r == nil {
return
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
info = &schedulerDetectionInfo{}
r.schedulerDetection[jobType] = info
}
info.lastDetectedAt = time.Now().UTC()
info.lastDetectedCount = count
info.lastError = ""
info.lastSkippedReason = ""
}
func (r *Plugin) recordSchedulerDetectionError(jobType string, err error) {
if r == nil {
return
}
if err == nil {
return
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
info = &schedulerDetectionInfo{}
r.schedulerDetection[jobType] = info
}
info.lastErrorAt = time.Now().UTC()
info.lastError = err.Error()
}
func (r *Plugin) recordSchedulerDetectionSkip(jobType string, reason string) {
if r == nil {
return
}
if strings.TrimSpace(reason) == "" {
return
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
info = &schedulerDetectionInfo{}
r.schedulerDetection[jobType] = info
}
info.lastSkippedAt = time.Now().UTC()
info.lastSkippedReason = reason
}
func (r *Plugin) snapshotSchedulerDetection(jobType string) schedulerDetectionInfo {
if r == nil {
return schedulerDetectionInfo{}
}
r.schedulerDetectionMu.Lock()
defer r.schedulerDetectionMu.Unlock()
info := r.schedulerDetection[jobType]
if info == nil {
return schedulerDetectionInfo{}
}
return *info
}
func (r *Plugin) GetSchedulerStatus() SchedulerStatus {
now := time.Now().UTC()
r.schedulerMu.Lock()
phase := r.schedulerPhase
currentJobType := r.currentJobType
iterationStartedAt := r.iterationStartedAt
lastIterationEndedAt := r.lastIterationEndedAt
lastIterationWorkDetected := r.lastIterationWorkDetected
r.schedulerMu.Unlock()
status := SchedulerStatus{
Now: now,
SchedulerTickSeconds: int(secondsFromDuration(r.idleSleepDuration)),
IdleSleepSeconds: int(secondsFromDuration(r.idleSleepDuration)),
Phase: phase,
CurrentJobType: currentJobType,
LastIterationWorkDetected: lastIterationWorkDetected,
InProcessJobs: r.listInProcessJobs(now),
}
if !iterationStartedAt.IsZero() {
status.IterationStartedAt = timeToPtr(iterationStartedAt)
}
if !lastIterationEndedAt.IsZero() {
status.LastIterationEndedAt = timeToPtr(lastIterationEndedAt)
}
states, err := r.ListSchedulerStates()
if err != nil {
return status
}
waiting := make([]SchedulerWaitingStatus, 0)
jobTypes := make([]SchedulerJobTypeStatus, 0, len(states))
for _, state := range states {
jobType := state.JobType
info := r.snapshotSchedulerDetection(jobType)
jobStatus := SchedulerJobTypeStatus{
JobType: jobType,
Enabled: state.Enabled,
DetectionInFlight: state.DetectionInFlight,
MaxJobTypeDurationSeconds: state.MaxJobTypeDurationSeconds,
}
if !info.lastDetectedAt.IsZero() {
jobStatus.LastDetectedAt = timeToPtr(info.lastDetectedAt)
jobStatus.LastDetectedCount = info.lastDetectedCount
}
if info.lastError != "" {
jobStatus.LastDetectionError = info.lastError
}
if info.lastSkippedReason != "" {
jobStatus.LastDetectionSkipped = info.lastSkippedReason
}
jobTypes = append(jobTypes, jobStatus)
if state.DetectionInFlight {
waiting = append(waiting, SchedulerWaitingStatus{
Reason: "processing",
JobType: jobType,
})
}
}
if phase == "idle" && !lastIterationEndedAt.IsZero() {
waiting = append(waiting, SchedulerWaitingStatus{
Reason: "idle_sleep",
})
}
sort.Slice(jobTypes, func(i, j int) bool {
return jobTypes[i].JobType < jobTypes[j].JobType
})
status.Waiting = waiting
status.JobTypes = jobTypes
return status
}
func (r *Plugin) listInProcessJobs(now time.Time) []SchedulerJobStatus {
active := make([]SchedulerJobStatus, 0)
if r == nil {
return active
}
r.jobsMu.RLock()
for _, job := range r.jobs {
if job == nil {
continue
}
if !isActiveTrackedJobState(job.State) {
continue
}
start := timeToPtr(now)
if job.CreatedAt != nil && !job.CreatedAt.IsZero() {
start = job.CreatedAt
} else if job.UpdatedAt != nil && !job.UpdatedAt.IsZero() {
start = job.UpdatedAt
}
durationSeconds := 0.0
if start != nil {
durationSeconds = now.Sub(*start).Seconds()
}
active = append(active, SchedulerJobStatus{
JobID: job.JobID,
JobType: job.JobType,
State: strings.ToLower(job.State),
Stage: job.Stage,
WorkerID: job.WorkerID,
Message: job.Message,
Progress: job.Progress,
CreatedAt: job.CreatedAt,
UpdatedAt: job.UpdatedAt,
DurationSeconds: durationSeconds,
})
}
r.jobsMu.RUnlock()
sort.Slice(active, func(i, j int) bool {
if active[i].DurationSeconds != active[j].DurationSeconds {
return active[i].DurationSeconds > active[j].DurationSeconds
}
return active[i].JobID < active[j].JobID
})
return active
}
@@ -1,64 +0,0 @@
package plugin
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
func TestGetSchedulerStatusIncludesInProcessJobs(t *testing.T) {
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
pluginSvc.trackExecutionStart("req-1", "worker-a", &plugin_pb.JobSpec{
JobId: "job-1",
JobType: "vacuum",
}, 1)
status := pluginSvc.GetSchedulerStatus()
if len(status.InProcessJobs) != 1 {
t.Fatalf("expected one in-process job, got %d", len(status.InProcessJobs))
}
if status.InProcessJobs[0].JobID != "job-1" {
t.Fatalf("unexpected job id: %s", status.InProcessJobs[0].JobID)
}
}
func TestGetSchedulerStatusIncludesLastDetectionCount(t *testing.T) {
pluginSvc, err := New(Options{})
if err != nil {
t.Fatalf("New: %v", err)
}
defer pluginSvc.Shutdown()
const jobType = "vacuum"
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
WorkerId: "worker-a",
Capabilities: []*plugin_pb.JobTypeCapability{
{JobType: jobType, CanDetect: true},
},
})
pluginSvc.recordSchedulerDetectionSuccess(jobType, 3)
status := pluginSvc.GetSchedulerStatus()
found := false
for _, jt := range status.JobTypes {
if jt.JobType != jobType {
continue
}
found = true
if jt.LastDetectedCount != 3 {
t.Fatalf("unexpected last detected count: got=%d want=3", jt.LastDetectedCount)
}
if jt.LastDetectedAt == nil {
t.Fatalf("expected last detected at to be set")
}
}
if !found {
t.Fatalf("expected job type status for %s", jobType)
}
}
+17 -21
View File
@@ -1,9 +1,6 @@
package plugin
import (
"errors"
"time"
)
import "time"
const (
// Keep exactly the last 10 successful and last 10 error runs per job type.
@@ -11,8 +8,6 @@ const (
MaxErrorRunHistory = 10
)
var ErrJobNotFound = errors.New("job not found")
type RunOutcome string
const (
@@ -82,21 +77,22 @@ type JobDetail struct {
}
type SchedulerJobTypeState struct {
JobType string `json:"job_type"`
Enabled bool `json:"enabled"`
PolicyError string `json:"policy_error,omitempty"`
DetectionInFlight bool `json:"detection_in_flight"`
DetectionTimeoutSeconds int32 `json:"detection_timeout_seconds,omitempty"`
ExecutionTimeoutSeconds int32 `json:"execution_timeout_seconds,omitempty"`
MaxJobTypeDurationSeconds int32 `json:"max_job_type_duration_seconds,omitempty"`
MaxJobsPerDetection int32 `json:"max_jobs_per_detection,omitempty"`
GlobalExecutionConcurrency int `json:"global_execution_concurrency,omitempty"`
PerWorkerExecutionConcurrency int `json:"per_worker_execution_concurrency,omitempty"`
RetryLimit int `json:"retry_limit,omitempty"`
RetryBackoffSeconds int32 `json:"retry_backoff_seconds,omitempty"`
DetectorAvailable bool `json:"detector_available"`
DetectorWorkerID string `json:"detector_worker_id,omitempty"`
ExecutorWorkerCount int `json:"executor_worker_count"`
JobType string `json:"job_type"`
Enabled bool `json:"enabled"`
PolicyError string `json:"policy_error,omitempty"`
DetectionInFlight bool `json:"detection_in_flight"`
NextDetectionAt *time.Time `json:"next_detection_at,omitempty"`
DetectionIntervalSeconds int32 `json:"detection_interval_seconds,omitempty"`
DetectionTimeoutSeconds int32 `json:"detection_timeout_seconds,omitempty"`
ExecutionTimeoutSeconds int32 `json:"execution_timeout_seconds,omitempty"`
MaxJobsPerDetection int32 `json:"max_jobs_per_detection,omitempty"`
GlobalExecutionConcurrency int `json:"global_execution_concurrency,omitempty"`
PerWorkerExecutionConcurrency int `json:"per_worker_execution_concurrency,omitempty"`
RetryLimit int `json:"retry_limit,omitempty"`
RetryBackoffSeconds int32 `json:"retry_backoff_seconds,omitempty"`
DetectorAvailable bool `json:"detector_available"`
DetectorWorkerID string `json:"detector_worker_id,omitempty"`
ExecutorWorkerCount int `json:"executor_worker_count"`
}
func timeToPtr(t time.Time) *time.Time {
-2
View File
@@ -115,7 +115,6 @@ func (at *ActiveTopology) GetDisksWithEffectiveCapacity(taskType TaskType, exclu
RemoteVolumeCount: disk.DiskInfo.DiskInfo.RemoteVolumeCount,
ActiveVolumeCount: disk.DiskInfo.DiskInfo.ActiveVolumeCount,
FreeVolumeCount: disk.DiskInfo.DiskInfo.FreeVolumeCount,
Tags: append([]string(nil), disk.DiskInfo.DiskInfo.Tags...),
}
diskCopy.DiskInfo = diskInfoCopy
diskCopy.DiskInfo.MaxVolumeCount = disk.DiskInfo.DiskInfo.MaxVolumeCount // Ensure Max is set
@@ -179,7 +178,6 @@ func (at *ActiveTopology) GetDisksForPlanning(taskType TaskType, excludeNodeID s
RemoteVolumeCount: disk.DiskInfo.DiskInfo.RemoteVolumeCount,
ActiveVolumeCount: disk.DiskInfo.DiskInfo.ActiveVolumeCount,
FreeVolumeCount: disk.DiskInfo.DiskInfo.FreeVolumeCount,
Tags: append([]string(nil), disk.DiskInfo.DiskInfo.Tags...),
}
diskCopy.DiskInfo = diskInfoCopy
+16 -164
View File
@@ -115,42 +115,12 @@ templ Plugin(page string) {
</div>
</div>
</div>
<div class="row mb-4">
<div class="col-12">
<div class="card shadow-sm">
<div class="card-header d-flex justify-content-between align-items-center flex-wrap gap-2">
<h5 class="mb-0"><i class="fas fa-sync-alt me-2"></i>Scheduler Iteration</h5>
<small class="text-muted">Current iteration phase and timing</small>
</div>
<div class="card-body" id="plugin-scheduler-iteration-card">
<div class="row g-3">
<div class="col-md-3 col-sm-6">
<div class="small text-muted">Phase</div>
<div class="fw-semibold" id="plugin-scheduler-phase">-</div>
</div>
<div class="col-md-3 col-sm-6">
<div class="small text-muted">Current Job Type</div>
<div class="fw-semibold" id="plugin-scheduler-current-jt">-</div>
</div>
<div class="col-md-3 col-sm-6">
<div class="small text-muted">Idle Sleep</div>
<div class="fw-semibold" id="plugin-scheduler-idle-sleep">-</div>
</div>
<div class="col-md-3 col-sm-6">
<div class="small text-muted">Last Iteration</div>
<div class="fw-semibold" id="plugin-scheduler-last-iteration">-</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="row mb-4">
<div class="col-12">
<div class="card shadow-sm">
<div class="card-header d-flex justify-content-between align-items-center flex-wrap gap-2">
<h5 class="mb-0"><i class="fas fa-clock me-2"></i>Scheduler State</h5>
<small class="text-muted">Per job type execution limits and status</small>
<small class="text-muted">Per job type detection schedule and execution limits</small>
</div>
<div class="card-body p-0">
<div class="table-responsive">
@@ -160,8 +130,9 @@ templ Plugin(page string) {
<th>Job Type</th>
<th>Enabled</th>
<th>Detector</th>
<th>Status</th>
<th>Max Duration</th>
<th>In Flight</th>
<th>Next Detection</th>
<th>Interval</th>
<th>Exec Global</th>
<th>Exec/Worker</th>
<th>Executor Workers</th>
@@ -169,7 +140,7 @@ templ Plugin(page string) {
</tr>
</thead>
<tbody id="plugin-scheduler-table-body">
<tr><td colspan="9" class="text-muted text-center py-3">Loading...</td></tr>
<tr><td colspan="10" class="text-muted text-center py-3">Loading...</td></tr>
</tbody>
</table>
</div>
@@ -514,11 +485,6 @@ templ Plugin(page string) {
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="plugin-job-detail-modal-label"><i class="fas fa-file-alt me-2"></i>Job Detail</h5>
<div class="ms-auto me-2">
<button type="button" class="btn btn-outline-danger btn-sm" id="plugin-expire-job-btn" disabled>
<i class="fas fa-stop-circle me-1"></i>Expire Job
</button>
</div>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body" id="plugin-job-detail-content">
@@ -540,9 +506,6 @@ templ Plugin(page string) {
.plugin-form-root .card {
border: 1px solid #dee2e6;
}
.plugin-form-root textarea {
min-height: 12rem;
}
.plugin-field-hidden {
display: none;
@@ -1110,66 +1073,6 @@ templ Plugin(page string) {
return html;
}
function isActiveJobState(candidateState) {
var jobState = candidateState;
if (candidateState && typeof candidateState === 'object' && candidateState.state !== undefined) {
jobState = candidateState.state;
}
var st = String(jobState || '').toLowerCase();
return st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' ||
st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
}
function setExpireButtonState(job) {
var expireBtn = document.getElementById('plugin-expire-job-btn');
if (!expireBtn) {
return;
}
var jobID = job && job.job_id ? String(job.job_id) : '';
var active = isActiveJobState(job);
expireBtn.setAttribute('data-job-id', jobID);
expireBtn.disabled = !jobID || !active;
if (!jobID) {
expireBtn.title = 'Select a job to expire.';
} else if (!active) {
expireBtn.title = 'Job is not active.';
} else {
expireBtn.title = 'Expire job to unblock scheduling.';
}
}
async function expireJob(jobID) {
var normalizedJobID = String(jobID || '').trim();
if (!normalizedJobID) {
return;
}
var reason = window.prompt('Expire job ' + normalizedJobID + '? Optional reason:', 'job expired by admin request');
if (reason === null) {
return;
}
var expireBtn = document.getElementById('plugin-expire-job-btn');
if (expireBtn) {
expireBtn.disabled = true;
}
try {
var response = await pluginRequest('POST', '/api/plugin/jobs/' + encodePath(normalizedJobID) + '/expire', {
reason: reason,
});
if (response && response.expired === false) {
notify(response.message || 'Job is not active.', 'info');
} else {
notify('Job expired: ' + normalizedJobID, 'success');
}
await refreshJobsAndActivities();
await openJobDetail(normalizedJobID);
} catch (e) {
notify('Failed to expire job: ' + e.message, 'error');
}
}
async function openJobDetail(jobID) {
var normalizedJobID = String(jobID || '').trim();
if (!normalizedJobID) {
@@ -1190,12 +1093,10 @@ templ Plugin(page string) {
modal.show();
}
setExpireButtonState(null);
contentRoot.innerHTML = '<div class="text-muted">Loading job detail...</div>';
try {
var detail = await pluginRequest('GET', '/api/plugin/jobs/' + encodePath(normalizedJobID) + '/detail?activity_limit=500&related_limit=20');
var job = (detail && detail.job) ? detail.job : {};
setExpireButtonState(job);
var runRecord = detail && detail.run_record ? detail.run_record : null;
var activities = (detail && Array.isArray(detail.activities)) ? detail.activities : [];
var relatedJobs = (detail && Array.isArray(detail.related_jobs)) ? detail.related_jobs : [];
@@ -1296,7 +1197,6 @@ templ Plugin(page string) {
contentRoot.innerHTML = html;
} catch (e) {
setExpireButtonState(null);
contentRoot.innerHTML = '<div class="alert alert-danger mb-0">Failed to load job detail: ' + escapeHtml(e.message) + '</div>';
}
}
@@ -1338,7 +1238,8 @@ templ Plugin(page string) {
var allActivities = Array.isArray(state.allActivities) ? state.allActivities : [];
var activeCount = allJobs.filter(function(job) {
return isActiveJobState(job);
var st = String(job.state || '').toLowerCase();
return st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' || st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
}).length;
document.getElementById('plugin-status-workers').textContent = String(state.workers.length);
@@ -1364,7 +1265,8 @@ templ Plugin(page string) {
if (!jobType) {
continue;
}
var isActive = isActiveJobState(job);
var st = String(job.state || '').toLowerCase();
var isActive = st === 'job_state_pending' || st === 'job_state_assigned' || st === 'job_state_running' || st === 'pending' || st === 'assigned' || st === 'running' || st === 'in_progress';
if (!isActive) {
continue;
}
@@ -1461,7 +1363,7 @@ templ Plugin(page string) {
var states = Array.isArray(state.schedulerStates) ? state.schedulerStates : [];
if (!states.length) {
tbody.innerHTML = '<tr><td colspan="9" class="text-muted text-center py-3">No scheduler state available</td></tr>';
tbody.innerHTML = '<tr><td colspan="10" class="text-muted text-center py-3">No scheduler state available</td></tr>';
return;
}
@@ -1471,8 +1373,8 @@ templ Plugin(page string) {
var enabled = !!item.enabled;
var inFlight = !!item.detection_in_flight;
var detector = item.detector_available ? textOrDash(item.detector_worker_id) : 'No detector';
var maxDurationSeconds = Number(item.max_job_type_duration_seconds || 0);
var maxDurationText = maxDurationSeconds > 0 ? (String(Math.round(maxDurationSeconds / 60)) + 'm') : '-';
var intervalSeconds = Number(item.detection_interval_seconds || 0);
var intervalText = intervalSeconds > 0 ? (String(intervalSeconds) + 's') : '-';
var globalExec = Number(item.global_execution_concurrency || 0);
var perWorkerExec = Number(item.per_worker_execution_concurrency || 0);
var executorWorkers = Number(item.executor_worker_count || 0);
@@ -1483,7 +1385,7 @@ templ Plugin(page string) {
var effectiveExecText = enabled ? String(effectiveExec) : '-';
var enabledBadge = enabled ? '<span class="badge bg-success">Enabled</span>' : '<span class="badge bg-secondary">Disabled</span>';
var statusBadge = inFlight ? '<span class="badge bg-warning text-dark">Processing</span>' : '<span class="badge bg-light text-dark">Idle</span>';
var inFlightBadge = inFlight ? '<span class="badge bg-warning text-dark">Yes</span>' : '<span class="badge bg-light text-dark">No</span>';
var policyErrorHtml = '';
if (item.policy_error) {
policyErrorHtml = '<div><small class="text-danger">' + escapeHtml(String(item.policy_error)) + '</small></div>';
@@ -1493,8 +1395,9 @@ templ Plugin(page string) {
'<td>' + escapeHtml(textOrDash(item.job_type)) + policyErrorHtml + '</td>' +
'<td>' + enabledBadge + '</td>' +
'<td><small>' + escapeHtml(detector) + '</small></td>' +
'<td>' + statusBadge + '</td>' +
'<td><small>' + escapeHtml(maxDurationText) + '</small></td>' +
'<td>' + inFlightBadge + '</td>' +
'<td><small>' + escapeHtml(parseTime(item.next_detection_at) || '-') + '</small></td>' +
'<td><small>' + escapeHtml(intervalText) + '</small></td>' +
'<td><small>' + escapeHtml(globalExecText) + '</small></td>' +
'<td><small>' + escapeHtml(perWorkerExecText) + '</small></td>' +
'<td><small>' + escapeHtml(executorWorkersText) + '</small></td>' +
@@ -2808,37 +2711,6 @@ templ Plugin(page string) {
}
}
function renderSchedulerIterationStatus(schedulerStatus) {
var phaseEl = document.getElementById('plugin-scheduler-phase');
var currentJtEl = document.getElementById('plugin-scheduler-current-jt');
var idleSleepEl = document.getElementById('plugin-scheduler-idle-sleep');
var lastIterEl = document.getElementById('plugin-scheduler-last-iteration');
if (!phaseEl) {
return;
}
var sched = (schedulerStatus && schedulerStatus.scheduler) ? schedulerStatus.scheduler : {};
var phase = String(sched.phase || 'unknown');
var phaseColors = { idle: 'bg-secondary', acquiring_lock: 'bg-info', processing: 'bg-primary' };
var phaseColor = phaseColors[phase] || 'bg-dark';
phaseEl.innerHTML = '<span class="badge ' + phaseColor + '">' + escapeHtml(phase) + '</span>';
currentJtEl.textContent = sched.current_job_type || '-';
var idleSleep = Number(sched.idle_sleep_seconds || 0);
idleSleepEl.textContent = idleSleep > 0 ? (String(Math.round(idleSleep / 60)) + ' min') : '-';
var lastEnded = parseTime(sched.last_iteration_ended_at);
var workDetected = !!sched.last_iteration_work_detected;
if (lastEnded) {
var workBadge = workDetected
? '<span class="badge bg-success ms-1">work found</span>'
: '<span class="badge bg-light text-dark ms-1">no work</span>';
lastIterEl.innerHTML = '<small>' + escapeHtml(lastEnded) + '</small>' + workBadge;
} else {
lastIterEl.textContent = '-';
}
}
async function refreshJobsAndActivities() {
var executionStateFilter = document.getElementById('plugin-monitor-job-state-filter');
@@ -2847,18 +2719,10 @@ templ Plugin(page string) {
var allJobsPromise = pluginRequest('GET', '/api/plugin/jobs?limit=500');
var allActivitiesPromise = pluginRequest('GET', '/api/plugin/activities?limit=500');
var schedulerPromise = pluginRequest('GET', '/api/plugin/scheduler-states');
var schedulerStatusPromise = pluginRequest('GET', '/api/plugin/scheduler-status').catch(function(e) { return e; });
var allJobs = await allJobsPromise;
var allActivities = await allActivitiesPromise;
var schedulerStates = await schedulerPromise;
var schedulerStatusResult = await schedulerStatusPromise;
var schedulerStatus = null;
if (schedulerStatusResult instanceof Error) {
console.error('Failed to fetch scheduler status:', schedulerStatusResult);
} else {
schedulerStatus = schedulerStatusResult;
}
state.jobs = Array.isArray(allJobs) ? allJobs : [];
state.activities = Array.isArray(allActivities) ? allActivities : [];
@@ -2870,7 +2734,6 @@ templ Plugin(page string) {
renderExecutionJobs();
renderExecutionActivities();
renderSchedulerStates();
renderSchedulerIterationStatus(schedulerStatus);
renderStatus();
renderJobTypeSummary();
}
@@ -2915,17 +2778,6 @@ templ Plugin(page string) {
});
}
var expireBtn = document.getElementById('plugin-expire-job-btn');
if (expireBtn) {
expireBtn.addEventListener('click', function() {
var jobID = String(expireBtn.getAttribute('data-job-id') || '').trim();
if (!jobID) {
return;
}
expireJob(jobID);
});
}
document.getElementById('plugin-refresh-all-btn').addEventListener('click', function() {
refreshAll();
});
File diff suppressed because one or more lines are too long
+1 -7
View File
@@ -45,7 +45,6 @@ type AdminOptions struct {
readOnlyPassword *string
dataDir *string
icebergPort *int
idleSleepSeconds *int
}
func init() {
@@ -61,7 +60,6 @@ func init() {
a.readOnlyUser = cmdAdmin.Flag.String("readOnlyUser", "", "read-only user username (optional, for view-only access)")
a.readOnlyPassword = cmdAdmin.Flag.String("readOnlyPassword", "", "read-only user password (optional, for view-only access; requires adminPassword to be set)")
a.icebergPort = cmdAdmin.Flag.Int("iceberg.port", 8181, "Iceberg REST Catalog port (0 to hide in UI)")
a.idleSleepSeconds = cmdAdmin.Flag.Int("scheduler.idleSleep", 0, "scheduler idle sleep in seconds between iterations when no work is found (0 = default 17 minutes)")
}
var cmdAdmin = &Command{
@@ -292,11 +290,7 @@ func startAdminServer(ctx context.Context, options AdminOptions, enableUI bool,
}
// Create admin server (plugin is always enabled)
var idleSleep time.Duration
if options.idleSleepSeconds != nil && *options.idleSleepSeconds > 0 {
idleSleep = time.Duration(*options.idleSleepSeconds) * time.Second
}
adminServer := dash.NewAdminServer(*options.master, nil, dataDir, icebergPort, idleSleep)
adminServer := dash.NewAdminServer(*options.master, nil, dataDir, icebergPort)
// Show discovered filers
filers := adminServer.GetAllFilers()
+1 -2
View File
@@ -45,7 +45,7 @@ const (
defaultMiniVolumeSizeMB = 128 // Default volume size for mini mode
maxVolumeSizeMB = 1024 // Maximum volume size in MB (1GB)
GrpcPortOffset = 10000 // Offset used to calculate gRPC port from HTTP port
defaultMiniPluginJobTypes = "vacuum,volume_balance,erasure_coding,admin_script"
defaultMiniPluginJobTypes = "vacuum,volume_balance,erasure_coding"
)
var (
@@ -204,7 +204,6 @@ func initMiniVolumeFlags() {
miniOptions.v.publicUrl = cmdMini.Flag.String("volume.publicUrl", "", "publicly accessible address")
miniOptions.v.indexType = cmdMini.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
miniOptions.v.diskType = cmdMini.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
miniOptions.v.tags = cmdMini.Flag.String("volume.tags", "", "comma-separated tag groups per data dir; each group uses ':' (e.g. fast:ssd,archive)")
miniOptions.v.fixJpgOrientation = cmdMini.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
miniOptions.v.readMode = cmdMini.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
miniOptions.v.compactionMBPerSecond = cmdMini.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
+2 -2
View File
@@ -7,7 +7,7 @@ func TestMiniDefaultPluginJobTypes(t *testing.T) {
if err != nil {
t.Fatalf("parsePluginWorkerJobTypes(mini default) err = %v", err)
}
if len(jobTypes) != 4 {
t.Fatalf("expected mini default job types to include 4 handlers, got %v", jobTypes)
if len(jobTypes) != 3 {
t.Fatalf("expected mini default job types to include 3 handlers, got %v", jobTypes)
}
}
+2 -10
View File
@@ -123,14 +123,6 @@ func TestParsePluginWorkerJobTypes(t *testing.T) {
if _, err = parsePluginWorkerJobTypes(" , "); err != nil {
t.Fatalf("expected empty list to resolve to default vacuum: %v", err)
}
jobTypes, err = parsePluginWorkerJobTypes("admin-script,script,admin_script")
if err != nil {
t.Fatalf("parsePluginWorkerJobTypes(admin script aliases) err = %v", err)
}
if len(jobTypes) != 1 || jobTypes[0] != "admin_script" {
t.Fatalf("expected admin_script alias to resolve, got %v", jobTypes)
}
}
func TestPluginWorkerDefaultJobTypes(t *testing.T) {
@@ -138,8 +130,8 @@ func TestPluginWorkerDefaultJobTypes(t *testing.T) {
if err != nil {
t.Fatalf("parsePluginWorkerJobTypes(default setting) err = %v", err)
}
if len(jobTypes) != 4 {
t.Fatalf("expected default job types to include 4 handlers, got %v", jobTypes)
if len(jobTypes) != 3 {
t.Fatalf("expected default job types to include 3 handlers, got %v", jobTypes)
}
}
-1
View File
@@ -137,7 +137,6 @@ func init() {
serverOptions.v.id = cmdServer.Flag.String("volume.id", "", "volume server id. If empty, default to ip:port")
serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
serverOptions.v.diskType = cmdServer.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
serverOptions.v.tags = cmdServer.Flag.String("volume.tags", "", "comma-separated tag groups per data dir; each group uses ':' (e.g. fast:ssd,archive)")
serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.")
serverOptions.v.readMode = cmdServer.Flag.String("volume.readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|read in remote node|redirect volume location'.")
serverOptions.v.compactionMBPerSecond = cmdServer.Flag.Int("volume.compactionMBps", 0, "limit compaction speed in mega bytes per second")
+1 -39
View File
@@ -53,7 +53,6 @@ type VolumeServerOptions struct {
whiteList []string
indexType *string
diskType *string
tags *string
fixJpgOrientation *bool
readMode *string
cpuProfile *string
@@ -95,7 +94,6 @@ func init() {
v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name")
v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
v.diskType = cmdVolume.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
v.tags = cmdVolume.Flag.String("tags", "", "comma-separated tag groups per data dir; each group uses ':' (e.g. fast:ssd,archive)")
v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", false, "Adjust jpg orientation when uploading.")
v.readMode = cmdVolume.Flag.String("readMode", "proxy", "[local|proxy|redirect] how to deal with non-local volume: 'not found|proxy to remote node|redirect volume location'.")
v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file")
@@ -221,12 +219,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
glog.Fatalf("%d directories by -dir, but only %d disk types is set by -disk", len(v.folders), len(diskTypes))
}
var tagsArg string
if v.tags != nil {
tagsArg = *v.tags
}
folderTags := parseVolumeTags(tagsArg, len(v.folders))
// security related white list configuration
v.whiteList = util.StringSplit(volumeWhiteListOption, ",")
@@ -277,7 +269,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
*v.ip, *v.port, *v.portGrpc, *v.publicUrl, volumeServerId,
v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes, folderTags,
v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
*v.idxFolder,
volumeNeedleMapKind,
v.masters, constants.VolumePulsePeriod, *v.dataCenter, *v.rack,
@@ -342,36 +334,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
}
func parseVolumeTags(tagsArg string, folderCount int) [][]string {
if folderCount <= 0 {
return nil
}
tagEntries := []string{}
if strings.TrimSpace(tagsArg) != "" {
tagEntries = strings.Split(tagsArg, ",")
}
folderTags := make([][]string, folderCount)
// If exactly one tag entry provided, replicate it to all folders
if len(tagEntries) == 1 {
normalized := util.NormalizeTagList(strings.Split(tagEntries[0], ":"))
for i := 0; i < folderCount; i++ {
folderTags[i] = append([]string(nil), normalized...)
}
} else {
// Otherwise, assign tags to folders that have explicit entries
for i := 0; i < folderCount; i++ {
if i < len(tagEntries) {
folderTags[i] = util.NormalizeTagList(strings.Split(tagEntries[i], ":"))
} else {
// Initialize remaining folders with empty tag slice
folderTags[i] = []string{}
}
}
}
return folderTags
}
func shutdown(publicHttpDown httpdown.Server, clusterHttpServer httpdown.Server, grpcS *grpc.Server, volumeServer *weed_server.VolumeServer) {
// firstly, stop the public http service to prevent from receiving new user request
+2 -3
View File
@@ -7,11 +7,11 @@ import (
)
var cmdWorker = &Command{
UsageLine: "worker -admin=<admin_server> [-id=<worker_id>] [-jobType=vacuum,volume_balance,erasure_coding,admin_script] [-workingDir=<path>] [-heartbeat=15s] [-reconnect=5s] [-maxDetect=1] [-maxExecute=4] [-metricsPort=<port>] [-metricsIp=<ip>] [-debug]",
UsageLine: "worker -admin=<admin_server> [-id=<worker_id>] [-jobType=vacuum,volume_balance,erasure_coding] [-workingDir=<path>] [-heartbeat=15s] [-reconnect=5s] [-maxDetect=1] [-maxExecute=4] [-metricsPort=<port>] [-metricsIp=<ip>] [-debug]",
Short: "start a plugin.proto worker process",
Long: `Start an external plugin worker using weed/pb/plugin.proto over gRPC.
This command provides vacuum, volume_balance, erasure_coding, and admin_script job type
This command provides vacuum, volume_balance, and erasure_coding job type
contracts with the plugin stream runtime, including descriptor delivery,
heartbeat/load reporting, detection, and execution.
@@ -25,7 +25,6 @@ Examples:
weed worker -admin=localhost:23646 -jobType=volume_balance
weed worker -admin=localhost:23646 -jobType=vacuum,volume_balance
weed worker -admin=localhost:23646 -jobType=erasure_coding
weed worker -admin=localhost:23646 -jobType=admin_script
weed worker -admin=admin.example.com:23646 -id=plugin-vacuum-a -heartbeat=10s
weed worker -admin=localhost:23646 -workingDir=/var/lib/seaweedfs-plugin
weed worker -admin=localhost:23646 -metricsPort=9327 -metricsIp=0.0.0.0
+1 -5
View File
@@ -23,7 +23,7 @@ import (
"google.golang.org/grpc"
)
const defaultPluginWorkerJobTypes = "vacuum,volume_balance,erasure_coding,admin_script"
const defaultPluginWorkerJobTypes = "vacuum,volume_balance,erasure_coding"
type pluginWorkerRunOptions struct {
AdminServer string
@@ -156,8 +156,6 @@ func buildPluginWorkerHandler(jobType string, dialOption grpc.DialOption, maxExe
return pluginworker.NewVolumeBalanceHandler(dialOption), nil
case "erasure_coding":
return pluginworker.NewErasureCodingHandler(dialOption, workingDir), nil
case "admin_script":
return pluginworker.NewAdminScriptHandler(dialOption), nil
default:
return nil, fmt.Errorf("unsupported plugin job type %q", canonicalJobType)
}
@@ -222,8 +220,6 @@ func canonicalPluginWorkerJobType(jobType string) (string, error) {
return "volume_balance", nil
case "erasure_coding", "erasure-coding", "erasure.coding", "ec":
return "erasure_coding", nil
case "admin_script", "admin-script", "admin.script", "script", "admin":
return "admin_script", nil
default:
return "", fmt.Errorf("unsupported plugin job type %q", jobType)
}
+2 -2
View File
@@ -7,7 +7,7 @@ func TestWorkerDefaultJobTypes(t *testing.T) {
if err != nil {
t.Fatalf("parsePluginWorkerJobTypes(default worker flag) err = %v", err)
}
if len(jobTypes) != 4 {
t.Fatalf("expected default worker job types to include 4 handlers, got %v", jobTypes)
if len(jobTypes) != 3 {
t.Fatalf("expected default worker job types to include 3 handlers, got %v", jobTypes)
}
}
-10
View File
@@ -25,7 +25,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"golang.org/x/sync/singleflight"
)
const (
@@ -55,7 +54,6 @@ type Filer struct {
Signature int32
FilerConf *FilerConf
RemoteStorage *FilerRemoteStorage
lazyFetchGroup singleflight.Group
Dlm *lock_manager.DistributedLockManager
MaxFilenameLength uint32
deletionQuit chan struct{}
@@ -377,14 +375,6 @@ func (f *Filer) FindEntry(ctx context.Context, p util.FullPath) (entry *Entry, e
}
}
if entry == nil && (err == nil || errors.Is(err, filer_pb.ErrNotFound)) {
if lazy, lazyErr := f.maybeLazyFetchFromRemote(ctx, p); lazyErr != nil {
glog.V(1).InfofCtx(ctx, "FindEntry lazy fetch %s: %v", p, lazyErr)
} else if lazy != nil {
return lazy, nil
}
}
return entry, err
}
-116
View File
@@ -1,116 +0,0 @@
package filer
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type lazyFetchContextKey struct{}
// maybeLazyFetchFromRemote is called by FindEntry when the store returns no
// entry for p. If p is under a remote-storage mount, it stats the remote
// object, builds a filer Entry from the result, and persists it via
// CreateEntry with SkipCheckParentDirectory so phantom parent directories
// under the mount are not required.
//
// On a CreateEntry failure after a successful StatFile the in-memory entry is
// still returned (availability over consistency); the singleflight key is
// forgotten so the next lookup retries the filer write.
//
// Returns nil without error when: p is not under a remote mount; the remote
// reports the object does not exist; or any other remote error occurs.
func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (*Entry, error) {
// Prevent recursive invocation: CreateEntry calls FindEntry, which would
// re-enter this function and deadlock on the singleflight key.
if ctx.Value(lazyFetchContextKey{}) != nil {
return nil, nil
}
if f.RemoteStorage == nil {
return nil, nil
}
mountDir, remoteLoc := f.RemoteStorage.FindMountDirectory(p)
if remoteLoc == nil {
return nil, nil
}
client, _, found := f.RemoteStorage.FindRemoteStorageClient(p)
if !found {
return nil, nil
}
relPath := strings.TrimPrefix(string(p), string(mountDir))
if relPath != "" && !strings.HasPrefix(relPath, "/") {
relPath = "/" + relPath
}
base := strings.TrimSuffix(remoteLoc.Path, "/")
remotePath := "/" + strings.TrimLeft(base+relPath, "/")
objectLoc := &remote_pb.RemoteStorageLocation{
Name: remoteLoc.Name,
Bucket: remoteLoc.Bucket,
Path: remotePath,
}
type lazyFetchResult struct {
entry *Entry
}
key := string(p)
val, err, _ := f.lazyFetchGroup.Do(key, func() (interface{}, error) {
remoteEntry, statErr := client.StatFile(objectLoc)
if statErr != nil {
if errors.Is(statErr, remote_storage.ErrRemoteObjectNotFound) {
glog.V(3).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s not found in remote", p)
} else {
glog.Warningf("maybeLazyFetchFromRemote: stat %s failed: %v", p, statErr)
}
return lazyFetchResult{nil}, nil
}
if remoteEntry == nil {
glog.V(3).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s StatFile returned nil entry", p)
return lazyFetchResult{nil}, nil
}
mtime := time.Unix(remoteEntry.RemoteMtime, 0)
entry := &Entry{
FullPath: p,
Attr: Attr{
Mtime: mtime,
Crtime: mtime,
Mode: 0644,
FileSize: uint64(remoteEntry.RemoteSize),
},
Remote: remoteEntry,
}
persistBaseCtx, cancelPersist := context.WithTimeout(context.Background(), 30*time.Second)
defer cancelPersist()
persistCtx := context.WithValue(persistBaseCtx, lazyFetchContextKey{}, true)
saveErr := f.CreateEntry(persistCtx, entry, false, false, nil, true, f.MaxFilenameLength)
if saveErr != nil {
glog.Warningf("maybeLazyFetchFromRemote: failed to persist filer entry for %s: %v", p, saveErr)
f.lazyFetchGroup.Forget(key)
}
return lazyFetchResult{entry}, nil
})
if err != nil {
return nil, err
}
result, ok := val.(lazyFetchResult)
if !ok {
return nil, fmt.Errorf("maybeLazyFetchFromRemote: unexpected singleflight result type %T for %s", val, p)
}
return result.entry, nil
}
-370
View File
@@ -1,370 +0,0 @@
package filer
import (
"context"
"errors"
"fmt"
"io"
"sync"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
// --- minimal FilerStore stub ---
type stubFilerStore struct {
mu sync.Mutex
entries map[string]*Entry
insertErr error
}
func newStubFilerStore() *stubFilerStore {
return &stubFilerStore{entries: make(map[string]*Entry)}
}
func (s *stubFilerStore) GetName() string { return "stub" }
func (s *stubFilerStore) Initialize(util.Configuration, string) error { return nil }
func (s *stubFilerStore) Shutdown() {}
func (s *stubFilerStore) BeginTransaction(ctx context.Context) (context.Context, error) {
return ctx, nil
}
func (s *stubFilerStore) CommitTransaction(context.Context) error { return nil }
func (s *stubFilerStore) RollbackTransaction(context.Context) error { return nil }
func (s *stubFilerStore) KvPut(context.Context, []byte, []byte) error { return nil }
func (s *stubFilerStore) KvGet(context.Context, []byte) ([]byte, error) {
return nil, ErrKvNotFound
}
func (s *stubFilerStore) KvDelete(context.Context, []byte) error { return nil }
func (s *stubFilerStore) DeleteFolderChildren(context.Context, util.FullPath) error { return nil }
func (s *stubFilerStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc ListEachEntryFunc) (string, error) {
return "", nil
}
func (s *stubFilerStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc ListEachEntryFunc) (string, error) {
return "", nil
}
func (s *stubFilerStore) InsertEntry(_ context.Context, entry *Entry) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.insertErr != nil {
return s.insertErr
}
s.entries[string(entry.FullPath)] = entry
return nil
}
func (s *stubFilerStore) UpdateEntry(_ context.Context, entry *Entry) error {
s.mu.Lock()
defer s.mu.Unlock()
s.entries[string(entry.FullPath)] = entry
return nil
}
func (s *stubFilerStore) FindEntry(_ context.Context, p util.FullPath) (*Entry, error) {
s.mu.Lock()
defer s.mu.Unlock()
if e, ok := s.entries[string(p)]; ok {
return e, nil
}
return nil, filer_pb.ErrNotFound
}
func (s *stubFilerStore) DeleteEntry(_ context.Context, p util.FullPath) error {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.entries, string(p))
return nil
}
// --- minimal RemoteStorageClient stub ---
type stubRemoteClient struct {
statResult *filer_pb.RemoteEntry
statErr error
}
func (c *stubRemoteClient) StatFile(*remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
return c.statResult, c.statErr
}
func (c *stubRemoteClient) Traverse(*remote_pb.RemoteStorageLocation, remote_storage.VisitFunc) error {
return nil
}
func (c *stubRemoteClient) ReadFile(*remote_pb.RemoteStorageLocation, int64, int64) ([]byte, error) {
return nil, nil
}
func (c *stubRemoteClient) WriteDirectory(*remote_pb.RemoteStorageLocation, *filer_pb.Entry) error {
return nil
}
func (c *stubRemoteClient) RemoveDirectory(*remote_pb.RemoteStorageLocation) error { return nil }
func (c *stubRemoteClient) WriteFile(*remote_pb.RemoteStorageLocation, *filer_pb.Entry, io.Reader) (*filer_pb.RemoteEntry, error) {
return nil, nil
}
func (c *stubRemoteClient) UpdateFileMetadata(*remote_pb.RemoteStorageLocation, *filer_pb.Entry, *filer_pb.Entry) error {
return nil
}
func (c *stubRemoteClient) DeleteFile(*remote_pb.RemoteStorageLocation) error { return nil }
func (c *stubRemoteClient) ListBuckets() ([]*remote_storage.Bucket, error) { return nil, nil }
func (c *stubRemoteClient) CreateBucket(string) error { return nil }
func (c *stubRemoteClient) DeleteBucket(string) error { return nil }
// --- stub RemoteStorageClientMaker ---
type stubClientMaker struct {
client remote_storage.RemoteStorageClient
}
func (m *stubClientMaker) Make(*remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
return m.client, nil
}
func (m *stubClientMaker) HasBucket() bool { return true }
// --- test filer factory ---
func newTestFiler(t *testing.T, store *stubFilerStore, rs *FilerRemoteStorage) *Filer {
t.Helper()
dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
mc := wdclient.NewMasterClient(
dialOption, "test", cluster.FilerType,
pb.ServerAddress("localhost:0"), "", "",
*pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{}),
)
f := &Filer{
RemoteStorage: rs,
Store: NewFilerStoreWrapper(store),
MaxFilenameLength: 255,
MasterClient: mc,
fileIdDeletionQueue: util.NewUnboundedQueue(),
LocalMetaLogBuffer: log_buffer.NewLogBuffer("test", time.Minute,
func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, func() {}),
}
return f
}
// registerStubMaker registers a stub RemoteStorageClientMaker for the given
// type string and returns a cleanup function that restores the previous maker.
func registerStubMaker(t *testing.T, storageType string, client remote_storage.RemoteStorageClient) func() {
t.Helper()
prev := remote_storage.RemoteStorageClientMakers[storageType]
remote_storage.RemoteStorageClientMakers[storageType] = &stubClientMaker{client: client}
return func() {
if prev != nil {
remote_storage.RemoteStorageClientMakers[storageType] = prev
} else {
delete(remote_storage.RemoteStorageClientMakers, storageType)
}
}
}
// --- tests ---
func TestMaybeLazyFetchFromRemote_HitsRemoteAndPersists(t *testing.T) {
const storageType = "stub_lazy_hit"
stub := &stubRemoteClient{
statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 1234},
}
defer registerStubMaker(t, storageType, stub)()
conf := &remote_pb.RemoteConf{Name: "mystore", Type: storageType}
rs := NewFilerRemoteStorage()
rs.storageNameToConf[conf.Name] = conf
rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
Name: "mystore",
Bucket: "mybucket",
Path: "/",
})
store := newStubFilerStore()
f := newTestFiler(t, store, rs)
entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/file.txt")
require.NoError(t, err)
require.NotNil(t, entry)
assert.Equal(t, util.FullPath("/buckets/mybucket/file.txt"), entry.FullPath)
assert.Equal(t, int64(1234), entry.Remote.RemoteSize)
assert.Equal(t, uint64(1234), entry.FileSize)
// entry must have been persisted in the store
stored, sErr := store.FindEntry(context.Background(), "/buckets/mybucket/file.txt")
require.NoError(t, sErr)
assert.Equal(t, int64(1234), stored.Remote.RemoteSize)
}
func TestMaybeLazyFetchFromRemote_NotUnderMount(t *testing.T) {
rs := NewFilerRemoteStorage()
store := newStubFilerStore()
f := newTestFiler(t, store, rs)
entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/not/a/mounted/path.txt")
require.NoError(t, err)
assert.Nil(t, entry)
}
func TestMaybeLazyFetchFromRemote_RemoteObjectNotFound(t *testing.T) {
const storageType = "stub_lazy_notfound"
stub := &stubRemoteClient{statErr: remote_storage.ErrRemoteObjectNotFound}
defer registerStubMaker(t, storageType, stub)()
conf := &remote_pb.RemoteConf{Name: "storenotfound", Type: storageType}
rs := NewFilerRemoteStorage()
rs.storageNameToConf[conf.Name] = conf
rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
Name: "storenotfound",
Bucket: "mybucket",
Path: "/",
})
store := newStubFilerStore()
f := newTestFiler(t, store, rs)
entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/missing.txt")
require.NoError(t, err)
assert.Nil(t, entry)
}
func TestMaybeLazyFetchFromRemote_CreateEntryFailureReturnsInMemoryEntry(t *testing.T) {
const storageType = "stub_lazy_saveerr"
stub := &stubRemoteClient{
statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 42},
}
defer registerStubMaker(t, storageType, stub)()
conf := &remote_pb.RemoteConf{Name: "storesaveerr", Type: storageType}
rs := NewFilerRemoteStorage()
rs.storageNameToConf[conf.Name] = conf
rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
Name: "storesaveerr",
Bucket: "mybucket",
Path: "/",
})
store := newStubFilerStore()
store.insertErr = errors.New("simulated store failure")
f := newTestFiler(t, store, rs)
// even with a store failure, the in-memory entry should be returned
entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/failfile.txt")
require.NoError(t, err)
require.NotNil(t, entry, "should return in-memory entry even when CreateEntry fails")
assert.Equal(t, int64(42), entry.Remote.RemoteSize)
}
func TestMaybeLazyFetchFromRemote_LongestPrefixMount(t *testing.T) {
// Register maker for the root mount
const typeRoot = "stub_lp_root"
stubRoot := &stubRemoteClient{statResult: &filer_pb.RemoteEntry{RemoteMtime: 1, RemoteSize: 10}}
defer registerStubMaker(t, typeRoot, stubRoot)()
// Register maker for the prefix mount
const typePrefix = "stub_lp_prefix"
stubPrefix := &stubRemoteClient{statResult: &filer_pb.RemoteEntry{RemoteMtime: 2, RemoteSize: 20}}
defer registerStubMaker(t, typePrefix, stubPrefix)()
rs := NewFilerRemoteStorage()
rs.storageNameToConf["rootstore"] = &remote_pb.RemoteConf{Name: "rootstore", Type: typeRoot}
rs.storageNameToConf["prefixstore"] = &remote_pb.RemoteConf{Name: "prefixstore", Type: typePrefix}
rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
Name: "rootstore", Bucket: "root-bucket", Path: "/",
})
rs.mapDirectoryToRemoteStorage("/buckets/mybucket/prefix", &remote_pb.RemoteStorageLocation{
Name: "prefixstore", Bucket: "prefix-bucket", Path: "/",
})
store := newStubFilerStore()
f := newTestFiler(t, store, rs)
// path under root mount only
entryRoot, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/file.txt")
require.NoError(t, err)
require.NotNil(t, entryRoot)
assert.Equal(t, int64(10), entryRoot.Remote.RemoteSize, "root mount should be used")
// path under nested (longer) mount — must prefer the longer prefix
entryPrefix, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/prefix/file.txt")
require.NoError(t, err)
require.NotNil(t, entryPrefix)
assert.Equal(t, int64(20), entryPrefix.Remote.RemoteSize, "nested mount should win (longest prefix)")
}
type countingRemoteClient struct {
stubRemoteClient
statCalls int
}
func (c *countingRemoteClient) StatFile(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
c.statCalls++
return c.stubRemoteClient.StatFile(loc)
}
func TestMaybeLazyFetchFromRemote_ContextGuardPreventsRecursion(t *testing.T) {
const storageType = "stub_lazy_guard"
countingStub := &countingRemoteClient{
stubRemoteClient: stubRemoteClient{
statResult: &filer_pb.RemoteEntry{RemoteMtime: 1, RemoteSize: 1},
},
}
defer registerStubMaker(t, storageType, countingStub)()
conf := &remote_pb.RemoteConf{Name: "guardstore", Type: storageType}
rs := NewFilerRemoteStorage()
rs.storageNameToConf[conf.Name] = conf
rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
Name: "guardstore",
Bucket: "mybucket",
Path: "/",
})
store := newStubFilerStore()
f := newTestFiler(t, store, rs)
guardCtx := context.WithValue(context.Background(), lazyFetchContextKey{}, true)
entry, err := f.maybeLazyFetchFromRemote(guardCtx, "/buckets/mybucket/file.txt")
require.NoError(t, err)
assert.Nil(t, entry)
assert.Equal(t, 0, countingStub.statCalls, "guard should prevent StatFile from being called")
}
func TestFindEntry_LazyFetchOnMiss(t *testing.T) {
const storageType = "stub_lazy_findentry"
stub := &stubRemoteClient{
statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 999},
}
defer registerStubMaker(t, storageType, stub)()
conf := &remote_pb.RemoteConf{Name: "findentrystore", Type: storageType}
rs := NewFilerRemoteStorage()
rs.storageNameToConf[conf.Name] = conf
rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
Name: "findentrystore",
Bucket: "mybucket",
Path: "/",
})
store := newStubFilerStore()
f := newTestFiler(t, store, rs)
// First lookup: store miss → lazy fetch
entry, err := f.FindEntry(context.Background(), "/buckets/mybucket/obj.txt")
require.NoError(t, err, fmt.Sprintf("unexpected err: %v", err))
require.NotNil(t, entry)
assert.Equal(t, uint64(999), entry.FileSize)
// Second lookup: now in store, no remote call needed
entry2, err2 := f.FindEntry(context.Background(), "/buckets/mybucket/obj.txt")
require.NoError(t, err2)
require.NotNil(t, entry2)
assert.Equal(t, uint64(999), entry2.FileSize)
}
-3
View File
@@ -82,9 +82,6 @@ func (rs *FilerRemoteStorage) mapDirectoryToRemoteStorage(dir util.FullPath, loc
rs.rules.Put([]byte(dir+"/"), loc)
}
// FindMountDirectory returns the mount directory and location for p. When multiple
// mounts match (e.g. /buckets/b and /buckets/b/prefix), ptrie MatchPrefix visits
// shorter prefixes first, so the last match is the longest prefix.
func (rs *FilerRemoteStorage) FindMountDirectory(p util.FullPath) (mountDir util.FullPath, remoteLocation *remote_pb.RemoteStorageLocation) {
rs.rules.MatchPrefix([]byte(p), func(key []byte, value *remote_pb.RemoteStorageLocation) bool {
mountDir = util.FullPath(string(key[:len(key)-1]))
-35
View File
@@ -4,7 +4,6 @@ import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/stretchr/testify/assert"
)
@@ -34,37 +33,3 @@ func TestFilerRemoteStorage_FindRemoteStorageClient(t *testing.T) {
_, _, found4 := rs.FindRemoteStorageClient("/a/b/cc")
assert.Equal(t, false, found4, "should not find storage client")
}
func TestFilerRemoteStorage_FindMountDirectory_LongestPrefixWins(t *testing.T) {
conf := &remote_pb.RemoteConf{Name: "store", Type: "s3"}
rs := NewFilerRemoteStorage()
rs.storageNameToConf[conf.Name] = conf
rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
Name: "store",
Bucket: "bucket-root",
Path: "/",
})
rs.mapDirectoryToRemoteStorage("/buckets/mybucket/prefix", &remote_pb.RemoteStorageLocation{
Name: "store",
Bucket: "bucket-prefix",
Path: "/",
})
tests := []struct {
path string
wantMount string
wantBucket string
}{
{"/buckets/mybucket/file.txt", "/buckets/mybucket", "bucket-root"},
{"/buckets/mybucket/prefix/file.txt", "/buckets/mybucket/prefix", "bucket-prefix"},
{"/buckets/mybucket/prefix/sub/file.txt", "/buckets/mybucket/prefix", "bucket-prefix"},
}
for _, tt := range tests {
mountDir, loc := rs.FindMountDirectory(util.FullPath(tt.path))
assert.Equal(t, util.FullPath(tt.wantMount), mountDir, "mount dir for %s", tt.path)
if assert.NotNil(t, loc, "location for %s", tt.path) {
assert.Equal(t, tt.wantBucket, loc.Bucket, "bucket for %s", tt.path)
}
}
}
+1 -9
View File
@@ -61,11 +61,6 @@ service Seaweed {
//////////////////////////////////////////////////
message DiskTag {
uint32 disk_id = 1;
repeated string tags = 2;
}
message Heartbeat {
string ip = 1;
uint32 port = 2;
@@ -94,8 +89,6 @@ message Heartbeat {
// state flags
volume_server_pb.VolumeServerState state = 23;
repeated DiskTag disk_tags = 24;
}
message HeartbeatResponse {
@@ -299,7 +292,6 @@ message DiskInfo {
repeated VolumeEcShardInformationMessage ec_shard_infos = 7;
int64 remote_volume_count = 8;
uint32 disk_id = 9;
repeated string tags = 10;
}
message DataNodeInfo {
string id = 1;
@@ -468,4 +460,4 @@ message RaftLeadershipTransferResponse {
}
message VolumeGrowResponse {
}
}
File diff suppressed because it is too large Load Diff
-1
View File
@@ -314,7 +314,6 @@ message ErasureCodingTaskConfig {
int32 quiet_for_seconds = 2; // Minimum quiet time before EC
int32 min_volume_size_mb = 3; // Minimum volume size for EC
string collection_filter = 4; // Only process volumes from specific collections
repeated string preferred_tags = 5; // Disk tags to prioritize for EC shard placement
}
// BalanceTaskConfig contains balance-specific configuration
+2 -11
View File
@@ -2589,7 +2589,6 @@ type ErasureCodingTaskConfig struct {
QuietForSeconds int32 `protobuf:"varint,2,opt,name=quiet_for_seconds,json=quietForSeconds,proto3" json:"quiet_for_seconds,omitempty"` // Minimum quiet time before EC
MinVolumeSizeMb int32 `protobuf:"varint,3,opt,name=min_volume_size_mb,json=minVolumeSizeMb,proto3" json:"min_volume_size_mb,omitempty"` // Minimum volume size for EC
CollectionFilter string `protobuf:"bytes,4,opt,name=collection_filter,json=collectionFilter,proto3" json:"collection_filter,omitempty"` // Only process volumes from specific collections
PreferredTags []string `protobuf:"bytes,5,rep,name=preferred_tags,json=preferredTags,proto3" json:"preferred_tags,omitempty"` // Disk tags to prioritize for EC shard placement
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -2652,13 +2651,6 @@ func (x *ErasureCodingTaskConfig) GetCollectionFilter() string {
return ""
}
func (x *ErasureCodingTaskConfig) GetPreferredTags() []string {
if x != nil {
return x.PreferredTags
}
return nil
}
// BalanceTaskConfig contains balance-specific configuration
type BalanceTaskConfig struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -3567,13 +3559,12 @@ const file_worker_proto_rawDesc = "" +
"\x10VacuumTaskConfig\x12+\n" +
"\x11garbage_threshold\x18\x01 \x01(\x01R\x10garbageThreshold\x12/\n" +
"\x14min_volume_age_hours\x18\x02 \x01(\x05R\x11minVolumeAgeHours\x120\n" +
"\x14min_interval_seconds\x18\x03 \x01(\x05R\x12minIntervalSeconds\"\xed\x01\n" +
"\x14min_interval_seconds\x18\x03 \x01(\x05R\x12minIntervalSeconds\"\xc6\x01\n" +
"\x17ErasureCodingTaskConfig\x12%\n" +
"\x0efullness_ratio\x18\x01 \x01(\x01R\rfullnessRatio\x12*\n" +
"\x11quiet_for_seconds\x18\x02 \x01(\x05R\x0fquietForSeconds\x12+\n" +
"\x12min_volume_size_mb\x18\x03 \x01(\x05R\x0fminVolumeSizeMb\x12+\n" +
"\x11collection_filter\x18\x04 \x01(\tR\x10collectionFilter\x12%\n" +
"\x0epreferred_tags\x18\x05 \x03(\tR\rpreferredTags\"n\n" +
"\x11collection_filter\x18\x04 \x01(\tR\x10collectionFilter\"n\n" +
"\x11BalanceTaskConfig\x12/\n" +
"\x13imbalance_threshold\x18\x01 \x01(\x01R\x12imbalanceThreshold\x12(\n" +
"\x10min_server_count\x18\x02 \x01(\x05R\x0eminServerCount\"I\n" +
-635
View File
@@ -1,635 +0,0 @@
package pluginworker
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"regexp"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/shell"
"google.golang.org/grpc"
"google.golang.org/protobuf/types/known/timestamppb"
)
const (
adminScriptJobType = "admin_script"
maxAdminScriptOutputBytes = 16 * 1024
defaultAdminScriptRunMins = 17
adminScriptDetectTickSecs = 60
)
const defaultAdminScript = `ec.balance -apply
fs.log.purge -daysAgo=7
volume.deleteEmpty -quietFor=24h -apply
volume.fix.replication -apply
s3.clean.uploads -timeAgo=24h`
var adminScriptTokenRegex = regexp.MustCompile(`'.*?'|".*?"|\S+`)
type AdminScriptHandler struct {
grpcDialOption grpc.DialOption
}
func NewAdminScriptHandler(grpcDialOption grpc.DialOption) *AdminScriptHandler {
return &AdminScriptHandler{grpcDialOption: grpcDialOption}
}
func (h *AdminScriptHandler) Capability() *plugin_pb.JobTypeCapability {
return &plugin_pb.JobTypeCapability{
JobType: adminScriptJobType,
CanDetect: true,
CanExecute: true,
MaxDetectionConcurrency: 1,
MaxExecutionConcurrency: 1,
DisplayName: "Admin Script",
Description: "Execute custom admin shell scripts",
Weight: 20,
}
}
func (h *AdminScriptHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
return &plugin_pb.JobTypeDescriptor{
JobType: adminScriptJobType,
DisplayName: "Admin Script",
Description: "Run custom admin shell scripts not covered by built-in job types",
Icon: "fas fa-terminal",
DescriptorVersion: 1,
AdminConfigForm: &plugin_pb.ConfigForm{
FormId: "admin-script-admin",
Title: "Admin Script Configuration",
Description: "Define the admin shell script to execute.",
Sections: []*plugin_pb.ConfigSection{
{
SectionId: "script",
Title: "Script",
Description: "Commands run sequentially by the admin script worker.",
Fields: []*plugin_pb.ConfigField{
{
Name: "script",
Label: "Script",
Description: "Admin shell commands to execute (one per line).",
HelpText: "Lock/unlock are handled by the admin server; omit explicit lock/unlock commands.",
Placeholder: "volume.balance -apply\nvolume.fix.replication -apply",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXTAREA,
Required: true,
},
{
Name: "run_interval_minutes",
Label: "Run Interval (minutes)",
Description: "Minimum interval between successful admin script runs.",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
Required: true,
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}},
},
},
},
},
DefaultValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: defaultAdminScript},
},
"run_interval_minutes": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: defaultAdminScriptRunMins},
},
},
},
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
Enabled: true,
DetectionIntervalSeconds: adminScriptDetectTickSecs,
DetectionTimeoutSeconds: 300,
MaxJobsPerDetection: 1,
GlobalExecutionConcurrency: 1,
PerWorkerExecutionConcurrency: 1,
RetryLimit: 0,
RetryBackoffSeconds: 30,
},
WorkerDefaultValues: map[string]*plugin_pb.ConfigValue{},
}
}
func (h *AdminScriptHandler) Detect(ctx context.Context, request *plugin_pb.RunDetectionRequest, sender DetectionSender) error {
if request == nil {
return fmt.Errorf("run detection request is nil")
}
if sender == nil {
return fmt.Errorf("detection sender is nil")
}
if request.JobType != "" && request.JobType != adminScriptJobType {
return fmt.Errorf("job type %q is not handled by admin_script worker", request.JobType)
}
script := normalizeAdminScript(readStringConfig(request.GetAdminConfigValues(), "script", ""))
scriptName := strings.TrimSpace(readStringConfig(request.GetAdminConfigValues(), "script_name", ""))
runIntervalMinutes := readAdminScriptRunIntervalMinutes(request.GetAdminConfigValues())
if shouldSkipDetectionByInterval(request.GetLastSuccessfulRun(), runIntervalMinutes*60) {
_ = sender.SendActivity(buildDetectorActivity(
"skipped_by_interval",
fmt.Sprintf("ADMIN SCRIPT: Detection skipped due to run interval (%dm)", runIntervalMinutes),
map[string]*plugin_pb.ConfigValue{
"run_interval_minutes": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(runIntervalMinutes)},
},
},
))
if err := sender.SendProposals(&plugin_pb.DetectionProposals{
JobType: adminScriptJobType,
Proposals: []*plugin_pb.JobProposal{},
HasMore: false,
}); err != nil {
return err
}
return sender.SendComplete(&plugin_pb.DetectionComplete{
JobType: adminScriptJobType,
Success: true,
TotalProposals: 0,
})
}
commands := parseAdminScriptCommands(script)
execCount := countExecutableCommands(commands)
if execCount == 0 {
_ = sender.SendActivity(buildDetectorActivity(
"no_script",
"ADMIN SCRIPT: No executable commands configured",
map[string]*plugin_pb.ConfigValue{
"command_count": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(execCount)},
},
},
))
if err := sender.SendProposals(&plugin_pb.DetectionProposals{
JobType: adminScriptJobType,
Proposals: []*plugin_pb.JobProposal{},
HasMore: false,
}); err != nil {
return err
}
return sender.SendComplete(&plugin_pb.DetectionComplete{
JobType: adminScriptJobType,
Success: true,
TotalProposals: 0,
})
}
proposal := buildAdminScriptProposal(script, scriptName, execCount)
proposals := []*plugin_pb.JobProposal{proposal}
hasMore := false
maxResults := int(request.MaxResults)
if maxResults > 0 && len(proposals) > maxResults {
proposals = proposals[:maxResults]
hasMore = true
}
if err := sender.SendProposals(&plugin_pb.DetectionProposals{
JobType: adminScriptJobType,
Proposals: proposals,
HasMore: hasMore,
}); err != nil {
return err
}
return sender.SendComplete(&plugin_pb.DetectionComplete{
JobType: adminScriptJobType,
Success: true,
TotalProposals: 1,
})
}
func (h *AdminScriptHandler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequest, sender ExecutionSender) error {
if request == nil || request.Job == nil {
return fmt.Errorf("execute job request is nil")
}
if sender == nil {
return fmt.Errorf("execution sender is nil")
}
if request.Job.JobType != "" && request.Job.JobType != adminScriptJobType {
return fmt.Errorf("job type %q is not handled by admin_script worker", request.Job.JobType)
}
script := normalizeAdminScript(readStringConfig(request.Job.Parameters, "script", ""))
scriptName := strings.TrimSpace(readStringConfig(request.Job.Parameters, "script_name", ""))
if script == "" {
script = normalizeAdminScript(readStringConfig(request.GetAdminConfigValues(), "script", ""))
}
if scriptName == "" {
scriptName = strings.TrimSpace(readStringConfig(request.GetAdminConfigValues(), "script_name", ""))
}
commands := parseAdminScriptCommands(script)
execCommands := filterExecutableCommands(commands)
if len(execCommands) == 0 {
return sender.SendCompleted(&plugin_pb.JobCompleted{
Success: false,
ErrorMessage: "no executable admin script commands configured",
})
}
commandEnv, cancel, err := h.buildAdminScriptCommandEnv(ctx, request.ClusterContext)
if err != nil {
return sender.SendCompleted(&plugin_pb.JobCompleted{
Success: false,
ErrorMessage: err.Error(),
})
}
defer cancel()
if err := sender.SendProgress(&plugin_pb.JobProgressUpdate{
JobId: request.Job.JobId,
JobType: request.Job.JobType,
State: plugin_pb.JobState_JOB_STATE_ASSIGNED,
ProgressPercent: 0,
Stage: "assigned",
Message: "admin script job accepted",
Activities: []*plugin_pb.ActivityEvent{
buildExecutorActivity("assigned", "admin script job accepted"),
},
}); err != nil {
return err
}
output := &limitedBuffer{maxBytes: maxAdminScriptOutputBytes}
executed := 0
errorMessages := make([]string, 0)
executedCommands := make([]string, 0, len(execCommands))
for _, cmd := range execCommands {
if ctx.Err() != nil {
errorMessages = append(errorMessages, ctx.Err().Error())
break
}
commandLine := formatAdminScriptCommand(cmd)
executedCommands = append(executedCommands, commandLine)
_, _ = fmt.Fprintf(output, "$ %s\n", commandLine)
found := false
for _, command := range shell.Commands {
if command.Name() != cmd.Name {
continue
}
found = true
if err := command.Do(cmd.Args, commandEnv, output); err != nil {
msg := fmt.Sprintf("%s: %v", cmd.Name, err)
errorMessages = append(errorMessages, msg)
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
State: plugin_pb.JobState_JOB_STATE_RUNNING,
ProgressPercent: percentProgress(executed+1, len(execCommands)),
Stage: "error",
Message: msg,
Activities: []*plugin_pb.ActivityEvent{
buildExecutorActivity("error", msg),
},
})
}
break
}
if !found {
msg := fmt.Sprintf("unknown admin command: %s", cmd.Name)
errorMessages = append(errorMessages, msg)
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
State: plugin_pb.JobState_JOB_STATE_RUNNING,
ProgressPercent: percentProgress(executed+1, len(execCommands)),
Stage: "error",
Message: msg,
Activities: []*plugin_pb.ActivityEvent{
buildExecutorActivity("error", msg),
},
})
}
executed++
progress := percentProgress(executed, len(execCommands))
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
State: plugin_pb.JobState_JOB_STATE_RUNNING,
ProgressPercent: progress,
Stage: "running",
Message: fmt.Sprintf("executed %d/%d command(s)", executed, len(execCommands)),
Activities: []*plugin_pb.ActivityEvent{
buildExecutorActivity("running", commandLine),
},
})
}
scriptHash := hashAdminScript(script)
resultSummary := fmt.Sprintf("admin script executed (%d command(s))", executed)
if scriptName != "" {
resultSummary = fmt.Sprintf("admin script %q executed (%d command(s))", scriptName, executed)
}
outputValues := map[string]*plugin_pb.ConfigValue{
"command_count": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(executed)},
},
"error_count": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(len(errorMessages))},
},
"script_hash": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: scriptHash},
},
}
if scriptName != "" {
outputValues["script_name"] = &plugin_pb.ConfigValue{
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: scriptName},
}
}
if len(executedCommands) > 0 {
outputValues["commands"] = &plugin_pb.ConfigValue{
Kind: &plugin_pb.ConfigValue_StringList{
StringList: &plugin_pb.StringList{Values: executedCommands},
},
}
}
if out := strings.TrimSpace(output.String()); out != "" {
outputValues["output"] = &plugin_pb.ConfigValue{
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: out},
}
}
if output.truncated {
outputValues["output_truncated"] = &plugin_pb.ConfigValue{
Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: true},
}
}
success := len(errorMessages) == 0 && ctx.Err() == nil
errorMessage := ""
if !success {
errorMessage = strings.Join(errorMessages, "; ")
if ctx.Err() != nil {
if errorMessage == "" {
errorMessage = ctx.Err().Error()
} else {
errorMessage = fmt.Sprintf("%s; %s", errorMessage, ctx.Err().Error())
}
}
}
return sender.SendCompleted(&plugin_pb.JobCompleted{
Success: success,
ErrorMessage: errorMessage,
Result: &plugin_pb.JobResult{
Summary: resultSummary,
OutputValues: outputValues,
},
Activities: []*plugin_pb.ActivityEvent{
buildExecutorActivity("completed", resultSummary),
},
CompletedAt: timestamppb.Now(),
})
}
func readAdminScriptRunIntervalMinutes(values map[string]*plugin_pb.ConfigValue) int {
runIntervalMinutes := int(readInt64Config(values, "run_interval_minutes", defaultAdminScriptRunMins))
if runIntervalMinutes <= 0 {
return defaultAdminScriptRunMins
}
return runIntervalMinutes
}
type adminScriptCommand struct {
Name string
Args []string
Raw string
}
func normalizeAdminScript(script string) string {
script = strings.ReplaceAll(script, "\r\n", "\n")
return strings.TrimSpace(script)
}
func parseAdminScriptCommands(script string) []adminScriptCommand {
script = normalizeAdminScript(script)
if script == "" {
return nil
}
lines := strings.Split(script, "\n")
commands := make([]adminScriptCommand, 0)
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
for _, chunk := range strings.Split(line, ";") {
chunk = strings.TrimSpace(chunk)
if chunk == "" {
continue
}
parts := adminScriptTokenRegex.FindAllString(chunk, -1)
if len(parts) == 0 {
continue
}
args := make([]string, 0, len(parts)-1)
for _, arg := range parts[1:] {
args = append(args, strings.Trim(arg, "\"'"))
}
commands = append(commands, adminScriptCommand{
Name: strings.TrimSpace(parts[0]),
Args: args,
Raw: chunk,
})
}
}
return commands
}
func filterExecutableCommands(commands []adminScriptCommand) []adminScriptCommand {
exec := make([]adminScriptCommand, 0, len(commands))
for _, cmd := range commands {
if cmd.Name == "" {
continue
}
if isAdminScriptLockCommand(cmd.Name) {
continue
}
exec = append(exec, cmd)
}
return exec
}
func countExecutableCommands(commands []adminScriptCommand) int {
count := 0
for _, cmd := range commands {
if cmd.Name == "" {
continue
}
if isAdminScriptLockCommand(cmd.Name) {
continue
}
count++
}
return count
}
func isAdminScriptLockCommand(name string) bool {
switch strings.ToLower(strings.TrimSpace(name)) {
case "lock", "unlock":
return true
default:
return false
}
}
func buildAdminScriptProposal(script, scriptName string, commandCount int) *plugin_pb.JobProposal {
scriptHash := hashAdminScript(script)
summary := "Run admin script"
if scriptName != "" {
summary = fmt.Sprintf("Run admin script: %s", scriptName)
}
detail := fmt.Sprintf("Admin script with %d command(s)", commandCount)
proposalID := fmt.Sprintf("admin-script-%s-%d", scriptHash[:8], time.Now().UnixNano())
labels := map[string]string{
"script_hash": scriptHash,
}
if scriptName != "" {
labels["script_name"] = scriptName
}
return &plugin_pb.JobProposal{
ProposalId: proposalID,
DedupeKey: "admin-script:" + scriptHash,
JobType: adminScriptJobType,
Priority: plugin_pb.JobPriority_JOB_PRIORITY_NORMAL,
Summary: summary,
Detail: detail,
Parameters: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: script},
},
"script_name": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: scriptName},
},
"script_hash": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: scriptHash},
},
"command_count": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: int64(commandCount)},
},
},
Labels: labels,
}
}
func (h *AdminScriptHandler) buildAdminScriptCommandEnv(
ctx context.Context,
clusterContext *plugin_pb.ClusterContext,
) (*shell.CommandEnv, context.CancelFunc, error) {
if clusterContext == nil {
return nil, nil, fmt.Errorf("cluster context is required")
}
masters := normalizeAddressList(clusterContext.MasterGrpcAddresses)
if len(masters) == 0 {
return nil, nil, fmt.Errorf("missing master addresses for admin script")
}
filerGroup := ""
mastersValue := strings.Join(masters, ",")
options := shell.ShellOptions{
Masters: &mastersValue,
GrpcDialOption: h.grpcDialOption,
FilerGroup: &filerGroup,
Directory: "/",
}
filers := normalizeAddressList(clusterContext.FilerGrpcAddresses)
if len(filers) > 0 {
options.FilerAddress = pb.ServerAddress(filers[0])
} else {
glog.V(1).Infof("admin script worker missing filer address; filer-dependent commands may fail")
}
commandEnv := shell.NewCommandEnv(&options)
commandEnv.ForceNoLock()
ctx, cancel := context.WithCancel(ctx)
go commandEnv.MasterClient.KeepConnectedToMaster(ctx)
return commandEnv, cancel, nil
}
func normalizeAddressList(addresses []string) []string {
normalized := make([]string, 0, len(addresses))
seen := make(map[string]struct{}, len(addresses))
for _, address := range addresses {
address = strings.TrimSpace(address)
if address == "" {
continue
}
if _, exists := seen[address]; exists {
continue
}
seen[address] = struct{}{}
normalized = append(normalized, address)
}
return normalized
}
func hashAdminScript(script string) string {
sum := sha256.Sum256([]byte(script))
return hex.EncodeToString(sum[:])
}
func formatAdminScriptCommand(cmd adminScriptCommand) string {
if len(cmd.Args) == 0 {
return cmd.Name
}
return fmt.Sprintf("%s %s", cmd.Name, strings.Join(cmd.Args, " "))
}
func percentProgress(done, total int) float64 {
if total <= 0 {
return 0
}
if done < 0 {
done = 0
}
if done > total {
done = total
}
return float64(done) / float64(total) * 100
}
type limitedBuffer struct {
buf bytes.Buffer
maxBytes int
truncated bool
}
func (b *limitedBuffer) Write(p []byte) (int, error) {
if b == nil {
return len(p), nil
}
if b.maxBytes <= 0 {
b.truncated = true
return len(p), nil
}
remaining := b.maxBytes - b.buf.Len()
if remaining <= 0 {
b.truncated = true
return len(p), nil
}
if len(p) > remaining {
_, _ = b.buf.Write(p[:remaining])
b.truncated = true
return len(p), nil
}
_, _ = b.buf.Write(p)
return len(p), nil
}
func (b *limitedBuffer) String() string {
if b == nil {
return ""
}
return b.buf.String()
}
@@ -1,100 +0,0 @@
package pluginworker
import (
"context"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestAdminScriptDescriptorDefaults(t *testing.T) {
descriptor := NewAdminScriptHandler(nil).Descriptor()
if descriptor == nil {
t.Fatalf("expected descriptor")
}
if descriptor.AdminRuntimeDefaults == nil {
t.Fatalf("expected admin runtime defaults")
}
if descriptor.AdminRuntimeDefaults.DetectionIntervalSeconds != adminScriptDetectTickSecs {
t.Fatalf("unexpected detection interval seconds: got=%d want=%d",
descriptor.AdminRuntimeDefaults.DetectionIntervalSeconds, adminScriptDetectTickSecs)
}
if descriptor.AdminConfigForm == nil {
t.Fatalf("expected admin config form")
}
runInterval := readInt64Config(descriptor.AdminConfigForm.DefaultValues, "run_interval_minutes", 0)
if runInterval != defaultAdminScriptRunMins {
t.Fatalf("unexpected run_interval_minutes default: got=%d want=%d", runInterval, defaultAdminScriptRunMins)
}
script := readStringConfig(descriptor.AdminConfigForm.DefaultValues, "script", "")
if strings.TrimSpace(script) == "" {
t.Fatalf("expected non-empty default script")
}
}
func TestAdminScriptDetectSkipsByRunInterval(t *testing.T) {
handler := NewAdminScriptHandler(nil)
sender := &recordingDetectionSender{}
err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
JobType: adminScriptJobType,
LastSuccessfulRun: timestamppb.New(time.Now().Add(-2 * time.Minute)),
AdminConfigValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: defaultAdminScript},
},
"run_interval_minutes": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 17},
},
},
}, sender)
if err != nil {
t.Fatalf("detect returned err = %v", err)
}
if sender.proposals == nil {
t.Fatalf("expected proposals message")
}
if len(sender.proposals.Proposals) != 0 {
t.Fatalf("expected zero proposals, got %d", len(sender.proposals.Proposals))
}
if sender.complete == nil || !sender.complete.Success {
t.Fatalf("expected successful completion message")
}
if len(sender.events) == 0 {
t.Fatalf("expected detector activity events")
}
if !strings.Contains(sender.events[0].Message, "run interval") {
t.Fatalf("unexpected skip message: %q", sender.events[0].Message)
}
}
func TestAdminScriptDetectCreatesProposalWhenIntervalElapsed(t *testing.T) {
handler := NewAdminScriptHandler(nil)
sender := &recordingDetectionSender{}
err := handler.Detect(context.Background(), &plugin_pb.RunDetectionRequest{
JobType: adminScriptJobType,
LastSuccessfulRun: timestamppb.New(time.Now().Add(-20 * time.Minute)),
AdminConfigValues: map[string]*plugin_pb.ConfigValue{
"script": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: defaultAdminScript},
},
"run_interval_minutes": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 17},
},
},
}, sender)
if err != nil {
t.Fatalf("detect returned err = %v", err)
}
if sender.proposals == nil {
t.Fatalf("expected proposals message")
}
if len(sender.proposals.Proposals) != 1 {
t.Fatalf("expected one proposal, got %d", len(sender.proposals.Proposals))
}
if sender.complete == nil || !sender.complete.Success || sender.complete.TotalProposals != 1 {
t.Fatalf("unexpected completion message: %+v", sender.complete)
}
}
@@ -11,7 +11,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
ecstorage "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
erasurecodingtask "github.com/seaweedfs/seaweedfs/weed/worker/tasks/erasure_coding"
@@ -129,14 +128,6 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
Required: true,
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
},
{
Name: "preferred_tags",
Label: "Preferred Tags",
Description: "Comma-separated disk tags to prioritize for EC shard placement, ordered by preference.",
Placeholder: "fast,ssd",
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING,
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT,
},
},
},
},
@@ -153,9 +144,6 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
"min_interval_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 60},
},
"preferred_tags": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
},
},
},
AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{
@@ -181,9 +169,6 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor {
"min_interval_seconds": {
Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 60},
},
"preferred_tags": {
Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""},
},
},
}
}
@@ -616,7 +601,6 @@ func deriveErasureCodingWorkerConfig(values map[string]*plugin_pb.ConfigValue) *
if minIntervalSeconds < 0 {
minIntervalSeconds = 0
}
taskConfig.PreferredTags = util.NormalizeTagList(readStringListConfig(values, "preferred_tags"))
return &erasureCodingWorkerConfig{
TaskConfig: taskConfig,
-1
View File
@@ -162,7 +162,6 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
}
dn.AdjustMaxVolumeCounts(heartbeat.MaxVolumeCounts)
dn.UpdateDiskTags(heartbeat.DiskTags)
glog.V(4).Infof("master received heartbeat %s", heartbeat.String())
stats.MasterReceivedHeartbeatCounter.WithLabelValues("total").Inc()
+2 -2
View File
@@ -58,7 +58,7 @@ type VolumeServer struct {
func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
port int, grpcPort int, publicUrl string, id string,
folders []string, maxCounts []int32, minFreeSpaces []util.MinFreeSpace, diskTypes []types.DiskType, diskTags [][]string,
folders []string, maxCounts []int32, minFreeSpaces []util.MinFreeSpace, diskTypes []types.DiskType,
idxFolder string,
needleMapKind storage.NeedleMapKind,
masterNodes []pb.ServerAddress, pulsePeriod time.Duration,
@@ -118,7 +118,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
vs.checkWithMaster()
vs.store = storage.NewStore(vs.grpcDialOption, ip, port, grpcPort, publicUrl, id, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes, diskTags, ldbTimeout)
vs.store = storage.NewStore(vs.grpcDialOption, ip, port, grpcPort, publicUrl, id, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes, ldbTimeout)
vs.guard = security.NewGuard(whiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec)
handleStaticResources(adminMux)
+2 -2
View File
@@ -2,9 +2,9 @@ package shell
import (
"flag"
"fmt"
"io"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
@@ -66,7 +66,7 @@ func (c *commandEcBalance) Do(args []string, commandEnv *CommandEnv, writer io.W
} else {
collections = append(collections, *collection)
}
glog.V(1).Infof("balanceEcVolumes collections %+v\n", len(collections))
fmt.Printf("balanceEcVolumes collections %+v\n", len(collections))
rp, err := parseReplicaPlacementArg(commandEnv, *shardReplicaPlacement)
if err != nil {
+3 -3
View File
@@ -150,14 +150,14 @@ func parseReplicaPlacementArg(commandEnv *CommandEnv, replicaStr string) (*super
if err != nil {
return rp, err
}
glog.V(1).Infof("using replica placement %q for EC volumes\n", rp.String())
fmt.Printf("using replica placement %q for EC volumes\n", rp.String())
} else {
// No replica placement argument provided, resolve from master default settings.
rp, err = getDefaultReplicaPlacement(commandEnv)
if err != nil {
return rp, err
}
glog.V(1).Infof("using master default replica placement %q for EC volumes\n", rp.String())
fmt.Printf("using master default replica placement %q for EC volumes\n", rp.String())
}
return rp, nil
@@ -1628,7 +1628,7 @@ func EcBalance(commandEnv *CommandEnv, collections []string, dc string, ecReplic
}
if len(collections) == 0 {
glog.V(1).Infof("WARNING: No collections to balance EC volumes across.\n")
fmt.Printf("WARNING: No collections to balance EC volumes across.\n")
}
for _, c := range collections {
if err = ecb.balanceEcVolumes(c); err != nil {
+8 -114
View File
@@ -35,7 +35,7 @@ func (c *commandEcDecode) Name() string {
func (c *commandEcDecode) Help() string {
return `decode a erasure coded volume into a normal volume
ec.decode [-collection=""] [-volumeId=<volume_id>] [-diskType=<disk_type>] [-checkMinFreeSpace]
ec.decode [-collection=""] [-volumeId=<volume_id>] [-diskType=<disk_type>]
The -collection parameter supports regular expressions for pattern matching:
- Use exact match: ec.decode -collection="^mybucket$"
@@ -44,7 +44,6 @@ func (c *commandEcDecode) Help() string {
Options:
-diskType: source disk type where EC shards are stored (hdd, ssd, or empty for default hdd)
-checkMinFreeSpace: check min free space when selecting the decode target (default true)
Examples:
# Decode EC shards from HDD (default)
@@ -65,7 +64,6 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
volumeId := decodeCommand.Int("volumeId", 0, "the volume id")
collection := decodeCommand.String("collection", "", "the collection name")
diskTypeStr := decodeCommand.String("diskType", "", "source disk type where EC shards are stored (hdd, ssd, or empty for default hdd)")
checkMinFreeSpace := decodeCommand.Bool("checkMinFreeSpace", true, "check min free space when selecting the decode target")
if err = decodeCommand.Parse(args); err != nil {
return nil
}
@@ -82,14 +80,10 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
if err != nil {
return err
}
var diskUsageState *decodeDiskUsageState
if *checkMinFreeSpace {
diskUsageState = newDecodeDiskUsageState(topologyInfo, diskType)
}
// volumeId is provided
if vid != 0 {
return doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState)
return doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType)
}
// apply to all volumes in the collection
@@ -99,7 +93,7 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
}
fmt.Printf("ec decode volumes: %v\n", volumeIds)
for _, vid := range volumeIds {
if err = doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType, *checkMinFreeSpace, diskUsageState); err != nil {
if err = doEcDecode(commandEnv, topologyInfo, *collection, vid, diskType); err != nil {
return err
}
}
@@ -107,7 +101,7 @@ func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Wr
return nil
}
func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId, diskType types.DiskType, checkMinFreeSpace bool, diskUsageState *decodeDiskUsageState) (err error) {
func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId, diskType types.DiskType) (err error) {
if !commandEnv.isLocked() {
return fmt.Errorf("lock is lost")
@@ -118,36 +112,8 @@ func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collec
fmt.Printf("ec volume %d shard locations: %+v\n", vid, nodeToEcShardsInfo)
if len(nodeToEcShardsInfo) == 0 {
return fmt.Errorf("no EC shards found for volume %d (diskType %s)", vid, diskType.ReadableString())
}
var originalShardCounts map[pb.ServerAddress]int
if diskUsageState != nil {
originalShardCounts = make(map[pb.ServerAddress]int, len(nodeToEcShardsInfo))
for location, si := range nodeToEcShardsInfo {
originalShardCounts[location] = si.Count()
}
}
var eligibleTargets map[pb.ServerAddress]struct{}
if checkMinFreeSpace {
if diskUsageState == nil {
return fmt.Errorf("min free space checking requires disk usage state")
}
eligibleTargets = make(map[pb.ServerAddress]struct{})
for location := range nodeToEcShardsInfo {
if freeCount, found := diskUsageState.freeVolumeCount(location); found && freeCount > 0 {
eligibleTargets[location] = struct{}{}
}
}
if len(eligibleTargets) == 0 {
return fmt.Errorf("no eligible target datanodes with free volume slots for volume %d (diskType %s); use -checkMinFreeSpace=false to override", vid, diskType.ReadableString())
}
}
// collect ec shards to the server with most space
targetNodeLocation, err := collectEcShards(commandEnv, nodeToEcShardsInfo, collection, vid, eligibleTargets)
targetNodeLocation, err := collectEcShards(commandEnv, nodeToEcShardsInfo, collection, vid)
if err != nil {
return fmt.Errorf("collectEcShards for volume %d: %v", vid, err)
}
@@ -158,13 +124,7 @@ func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collec
// Special case: if the EC index has no live entries, decoding is a no-op.
// Just purge EC shards and return success without generating/mounting an empty volume.
if isEcDecodeEmptyVolumeErr(err) {
if err := unmountAndDeleteEcShards(commandEnv.option.GrpcDialOption, collection, nodeToEcShardsInfo, vid); err != nil {
return err
}
if diskUsageState != nil {
diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, false)
}
return nil
return unmountAndDeleteEcShards(commandEnv.option.GrpcDialOption, collection, nodeToEcShardsInfo, vid)
}
return fmt.Errorf("generate normal volume %d on %s: %v", vid, targetNodeLocation, err)
}
@@ -174,9 +134,6 @@ func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collec
if err != nil {
return fmt.Errorf("delete ec shards for volume %d: %v", vid, err)
}
if diskUsageState != nil {
diskUsageState.applyDecode(targetNodeLocation, originalShardCounts, true)
}
return nil
}
@@ -250,16 +207,11 @@ func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, c
}
func collectEcShards(commandEnv *CommandEnv, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, collection string, vid needle.VolumeId, eligibleTargets map[pb.ServerAddress]struct{}) (targetNodeLocation pb.ServerAddress, err error) {
func collectEcShards(commandEnv *CommandEnv, nodeToShardsInfo map[pb.ServerAddress]*erasure_coding.ShardsInfo, collection string, vid needle.VolumeId) (targetNodeLocation pb.ServerAddress, err error) {
maxShardCount := -1
maxShardCount := 0
existingShardsInfo := erasure_coding.NewShardsInfo()
for loc, si := range nodeToShardsInfo {
if eligibleTargets != nil {
if _, ok := eligibleTargets[loc]; !ok {
continue
}
}
toBeCopiedShardCount := si.MinusParityShards().Count()
if toBeCopiedShardCount > maxShardCount {
maxShardCount = toBeCopiedShardCount
@@ -267,9 +219,6 @@ func collectEcShards(commandEnv *CommandEnv, nodeToShardsInfo map[pb.ServerAddre
existingShardsInfo = si
}
}
if targetNodeLocation == "" {
return "", fmt.Errorf("no eligible target datanodes available to decode volume %d", vid)
}
fmt.Printf("collectEcShards: ec volume %d collect shards to %s from: %+v\n", vid, targetNodeLocation, nodeToShardsInfo)
@@ -377,58 +326,3 @@ func collectEcNodeShardsInfo(topoInfo *master_pb.TopologyInfo, vid needle.Volume
return res
}
type decodeDiskUsageState struct {
byNode map[pb.ServerAddress]*decodeDiskUsageCounts
}
type decodeDiskUsageCounts struct {
maxVolumeCount int64
volumeCount int64
remoteVolumeCount int64
ecShardCount int64
}
func newDecodeDiskUsageState(topoInfo *master_pb.TopologyInfo, diskType types.DiskType) *decodeDiskUsageState {
state := &decodeDiskUsageState{byNode: make(map[pb.ServerAddress]*decodeDiskUsageCounts)}
eachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
if diskInfo, found := dn.DiskInfos[string(diskType)]; found {
state.byNode[pb.NewServerAddressFromDataNode(dn)] = &decodeDiskUsageCounts{
maxVolumeCount: diskInfo.MaxVolumeCount,
volumeCount: diskInfo.VolumeCount,
remoteVolumeCount: diskInfo.RemoteVolumeCount,
ecShardCount: int64(countShards(diskInfo.EcShardInfos)),
}
}
})
return state
}
func (state *decodeDiskUsageState) freeVolumeCount(location pb.ServerAddress) (int64, bool) {
if state == nil {
return 0, false
}
usage, found := state.byNode[location]
if !found {
return 0, false
}
free := usage.maxVolumeCount - (usage.volumeCount - usage.remoteVolumeCount)
free -= (usage.ecShardCount + int64(erasure_coding.DataShardsCount) - 1) / int64(erasure_coding.DataShardsCount)
return free, true
}
func (state *decodeDiskUsageState) applyDecode(targetNodeLocation pb.ServerAddress, shardCounts map[pb.ServerAddress]int, createdVolume bool) {
if state == nil {
return
}
for location, shardCount := range shardCounts {
if usage, found := state.byNode[location]; found {
usage.ecShardCount -= int64(shardCount)
}
}
if createdVolume {
if usage, found := state.byNode[targetNodeLocation]; found {
usage.volumeCount++
}
}
}
-43
View File
@@ -1,43 +0,0 @@
package shell
import (
"fmt"
"io"
"strconv"
"time"
)
func init() {
Commands = append(Commands, &commandSleep{})
}
// =========== Sleep ==============
type commandSleep struct {
}
func (c *commandSleep) Name() string {
return "sleep"
}
func (c *commandSleep) Help() string {
return `sleep for N seconds (useful to simulate long running jobs)
sleep 5
`
}
func (c *commandSleep) HasTag(CommandTag) bool {
return false
}
func (c *commandSleep) Do(args []string, _ *CommandEnv, _ io.Writer) error {
if len(args) == 0 {
return fmt.Errorf("sleep requires a seconds argument")
}
seconds, err := strconv.Atoi(args[0])
if err != nil || seconds <= 0 {
return fmt.Errorf("sleep duration must be a positive integer, got %q", args[0])
}
time.Sleep(time.Duration(seconds) * time.Second)
return nil
}
+1 -19
View File
@@ -38,7 +38,6 @@ type CommandEnv struct {
option *ShellOptions
locker *exclusive_locks.ExclusiveLocker
noLock bool
forceNoLock bool
verbose bool
}
@@ -72,9 +71,6 @@ func (ce *CommandEnv) isDirectory(path string) bool {
func (ce *CommandEnv) confirmIsLocked(args []string) error {
if ce.noLock || ce.forceNoLock {
return nil
}
if ce.locker.IsLocked() {
return nil
}
@@ -84,25 +80,11 @@ func (ce *CommandEnv) confirmIsLocked(args []string) error {
}
func (ce *CommandEnv) SetNoLock(noLock bool) {
if ce == nil {
return
}
ce.noLock = noLock
}
func (ce *CommandEnv) ForceNoLock() {
if ce == nil {
return
}
ce.forceNoLock = true
}
func (ce *CommandEnv) isLocked() bool {
if ce == nil {
return true
}
if ce.noLock || ce.forceNoLock {
if ce.noLock {
return true
}
return ce.locker.IsLocked()
+1 -9
View File
@@ -31,7 +31,6 @@ type DiskLocation struct {
DirectoryUuid string
IdxDirectory string
DiskType types.DiskType
Tags []string
MaxVolumeCount int32
OriginalMaxVolumeCount int32
MinFreeSpace util.MinFreeSpace
@@ -77,7 +76,7 @@ func writeNewUuid(fileName string) (string, error) {
return dirUuidString, nil
}
func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType, tags []string) *DiskLocation {
func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation {
glog.V(4).Infof("Added new Disk %s: maxVolumes=%d", dir, maxVolumeCount)
dir = util.ResolvePath(dir)
if idxDir == "" {
@@ -89,18 +88,11 @@ func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFree
if err != nil {
glog.Fatalf("cannot generate uuid of dir %s: %v", dir, err)
}
// Defensive copy of tags to prevent external mutation
var copiedTags []string
if len(tags) > 0 {
copiedTags = make([]string, len(tags))
copy(copiedTags, tags)
}
location := &DiskLocation{
Directory: dir,
DirectoryUuid: dirUuid,
IdxDirectory: idxDir,
DiskType: diskType,
Tags: copiedTags,
MaxVolumeCount: maxVolumeCount,
OriginalMaxVolumeCount: maxVolumeCount,
MinFreeSpace: minFreeSpace,
+1 -14
View File
@@ -92,7 +92,6 @@ func NewStore(
idxFolder string,
needleMapKind NeedleMapKind,
diskTypes []DiskType,
diskTags [][]string,
ldbTimeout int64,
) (s *Store) {
s = &Store{
@@ -114,11 +113,7 @@ func NewStore(
var wg sync.WaitGroup
for i := 0; i < len(dirnames); i++ {
var tags []string
if i < len(diskTags) {
tags = diskTags[i]
}
location := NewDiskLocation(dirnames[i], int32(maxVolumeCounts[i]), minFreeSpaces[i], idxFolder, diskTypes[i], tags)
location := NewDiskLocation(dirnames[i], int32(maxVolumeCounts[i]), minFreeSpaces[i], idxFolder, diskTypes[i])
s.Locations = append(s.Locations, location)
stats.VolumeServerMaxVolumeCounter.Add(float64(maxVolumeCounts[i]))
@@ -479,13 +474,6 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
for _, loc := range s.Locations {
uuidList = append(uuidList, loc.DirectoryUuid)
}
var diskTags []*master_pb.DiskTag
for diskID, loc := range s.Locations {
diskTags = append(diskTags, &master_pb.DiskTag{
DiskId: uint32(diskID),
Tags: append([]string(nil), loc.Tags...),
})
}
for col, size := range collectionVolumeSize {
stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "normal").Set(float64(size))
@@ -516,7 +504,6 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
HasNoVolumes: len(volumeMessages) == 0,
HasNoEcShards: len(ecVolumeMessages) == 0,
LocationUuids: uuidList,
DiskTags: diskTags,
}
}
+1 -1
View File
@@ -32,7 +32,7 @@ func newTestStore(t *testing.T, numDirs int) *Store {
}
store := NewStore(nil, "localhost", 8080, 18080, "http://localhost:8080", "",
dirs, maxCounts, minFreeSpaces, "", NeedleMapInMemory, diskTypes, nil, 3)
dirs, maxCounts, minFreeSpaces, "", NeedleMapInMemory, diskTypes, 3)
// Consume channel messages to prevent blocking
done := make(chan bool)
-33
View File
@@ -25,7 +25,6 @@ type DataNode struct {
IsTerminating bool
MaintenanceMode bool
diskTags map[uint32][]string
}
func NewDataNode(id string) *DataNode {
@@ -292,41 +291,9 @@ func (dn *DataNode) ToDataNodeInfo() *master_pb.DataNodeInfo {
disk := c.(*Disk)
m.DiskInfos[string(disk.Id())] = disk.ToDiskInfo()
}
dn.RLock()
diskTags := make(map[uint32][]string, len(dn.diskTags))
for diskID, tags := range dn.diskTags {
diskTags[diskID] = append([]string(nil), tags...)
}
dn.RUnlock()
for _, diskInfo := range m.DiskInfos {
if diskInfo == nil {
continue
}
if tags, found := diskTags[diskInfo.DiskId]; found {
diskInfo.Tags = append([]string(nil), tags...)
}
}
return m
}
func (dn *DataNode) UpdateDiskTags(tags []*master_pb.DiskTag) {
if len(tags) == 0 {
return
}
dn.Lock()
if dn.diskTags == nil {
dn.diskTags = make(map[uint32][]string, len(tags))
}
for _, tagInfo := range tags {
if tagInfo == nil {
continue
}
dn.diskTags[tagInfo.DiskId] = append([]string(nil), tagInfo.Tags...)
}
dn.Unlock()
}
// GetVolumeIds returns the human readable volume ids limited to count of max 100.
func (dn *DataNode) GetVolumeIds() string {
dn.RLock()
-25
View File
@@ -1,25 +0,0 @@
package util
import "strings"
// NormalizeTagList normalizes a list of tags by converting to lowercase,
// trimming whitespace, removing duplicates, and filtering empty strings.
func NormalizeTagList(tags []string) []string {
normalized := make([]string, 0, len(tags))
seen := make(map[string]struct{}, len(tags))
for _, tag := range tags {
tag = strings.ToLower(strings.TrimSpace(tag))
if tag == "" {
continue
}
if _, exists := seen[tag]; exists {
continue
}
seen[tag] = struct{}{}
normalized = append(normalized, tag)
}
if len(normalized) == 0 {
return []string{}
}
return normalized
}
+6 -46
View File
@@ -11,8 +11,6 @@ import (
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
@@ -28,22 +26,6 @@ type masterVolumeProvider struct {
masterClient *MasterClient
}
func isCanceledErr(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return true
}
if statusErr, ok := status.FromError(err); ok {
switch statusErr.Code() {
case codes.Canceled, codes.DeadlineExceeded:
return true
}
}
return false
}
// LookupVolumeIds queries the master for volume locations (fallback when cache misses)
// Returns partial results with aggregated errors for volumes that failed
func (p *masterVolumeProvider) LookupVolumeIds(ctx context.Context, volumeIds []string) (map[string][]Location, error) {
@@ -212,13 +194,8 @@ func (mc *MasterClient) tryConnectToMaster(ctx context.Context, master pb.Server
resp, err := stream.Recv()
if err != nil {
canceled := isCanceledErr(err) || ctx.Err() != nil
if canceled {
glog.V(1).Infof("%s.%s masterClient stream closed from %s: %v", mc.FilerGroup, mc.clientType, master, err)
} else {
glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
}
glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
return err
}
@@ -242,13 +219,8 @@ func (mc *MasterClient) tryConnectToMaster(ctx context.Context, master pb.Server
for {
resp, err := stream.Recv()
if err != nil {
canceled := isCanceledErr(err) || ctx.Err() != nil
if canceled {
glog.V(1).Infof("%s.%s masterClient stream closed from %s: %v", mc.FilerGroup, mc.clientType, master, err)
} else {
glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
}
glog.V(0).Infof("%s.%s masterClient failed to receive from %s: %v", mc.FilerGroup, mc.clientType, master, err)
stats.MasterClientConnectCounter.WithLabelValues(stats.FailedToReceive).Inc()
return err
}
@@ -280,20 +252,12 @@ func (mc *MasterClient) tryConnectToMaster(ctx context.Context, master pb.Server
mc.OnPeerUpdateLock.RUnlock()
}
if err := ctx.Err(); err != nil {
if isCanceledErr(err) {
glog.V(1).Infof("Connection attempt to master stopped: %v", err)
} else {
glog.V(0).Infof("Connection attempt to master stopped: %v", err)
}
glog.V(0).Infof("Connection attempt to master stopped: %v", err)
return err
}
}
})
if gprcErr != nil {
if isCanceledErr(gprcErr) || ctx.Err() != nil {
glog.V(1).Infof("%s.%s masterClient connection closed to %v: %v", mc.FilerGroup, mc.clientType, master, gprcErr)
return nextHintedLeader
}
stats.MasterClientConnectCounter.WithLabelValues(stats.Failed).Inc()
glog.V(1).Infof("%s.%s masterClient failed to connect with master %v: %v", mc.FilerGroup, mc.clientType, master, gprcErr)
}
@@ -423,11 +387,7 @@ func (mc *MasterClient) KeepConnectedToMaster(ctx context.Context) {
for {
select {
case <-ctx.Done():
if isCanceledErr(ctx.Err()) {
glog.V(1).Infof("Connection to masters stopped: %v", ctx.Err())
} else {
glog.V(0).Infof("Connection to masters stopped: %v", ctx.Err())
}
glog.V(0).Infof("Connection to masters stopped: %v", ctx.Err())
return
default:
reconnectStart := time.Now()
+4 -23
View File
@@ -12,11 +12,10 @@ import (
// Config extends BaseConfig with erasure coding specific settings
type Config struct {
base.BaseConfig
QuietForSeconds int `json:"quiet_for_seconds"`
FullnessRatio float64 `json:"fullness_ratio"`
CollectionFilter string `json:"collection_filter"`
MinSizeMB int `json:"min_size_mb"`
PreferredTags []string `json:"preferred_tags"`
QuietForSeconds int `json:"quiet_for_seconds"`
FullnessRatio float64 `json:"fullness_ratio"`
CollectionFilter string `json:"collection_filter"`
MinSizeMB int `json:"min_size_mb"`
}
// NewDefaultConfig creates a new default erasure coding configuration
@@ -31,7 +30,6 @@ func NewDefaultConfig() *Config {
FullnessRatio: 0.8, // 80%
CollectionFilter: "",
MinSizeMB: 30, // 30MB (more reasonable than 100MB)
PreferredTags: nil,
}
}
@@ -144,27 +142,12 @@ func GetConfigSpec() base.ConfigSpec {
InputType: "number",
CSSClasses: "form-control",
},
{
Name: "preferred_tags",
JSONName: "preferred_tags",
Type: config.FieldTypeString,
DefaultValue: "",
Required: false,
DisplayName: "Preferred Disk Tags",
Description: "Comma-separated disk tags to prioritize for EC shard placement",
HelpText: "EC shards will be placed on disks with these tags first, then fall back to other disks if needed",
Placeholder: "fast,ssd",
InputType: "text",
CSSClasses: "form-control",
},
},
}
}
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
// Defensive copy of PreferredTags to prevent external mutation
preferredTagsCopy := append([]string(nil), c.PreferredTags...)
return &worker_pb.TaskPolicy{
Enabled: c.Enabled,
MaxConcurrent: int32(c.MaxConcurrent),
@@ -176,7 +159,6 @@ func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
QuietForSeconds: int32(c.QuietForSeconds),
MinVolumeSizeMb: int32(c.MinSizeMB),
CollectionFilter: c.CollectionFilter,
PreferredTags: preferredTagsCopy,
},
},
}
@@ -199,7 +181,6 @@ func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
c.QuietForSeconds = int(ecConfig.QuietForSeconds)
c.MinSizeMB = int(ecConfig.MinVolumeSizeMb)
c.CollectionFilter = ecConfig.CollectionFilter
c.PreferredTags = append([]string(nil), ecConfig.PreferredTags...)
}
return nil
+9 -99
View File
@@ -11,10 +11,9 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding/placement"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
workerutil "github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/util"
"github.com/seaweedfs/seaweedfs/weed/worker/types"
)
@@ -149,7 +148,7 @@ func Detection(ctx context.Context, metrics []*types.VolumeHealthMetrics, cluste
glog.Infof("EC Detection: ActiveTopology available, planning destinations for volume %d", metric.VolumeID)
if planner == nil {
planner = newECPlacementPlanner(clusterInfo.ActiveTopology, ecConfig.PreferredTags)
planner = newECPlacementPlanner(clusterInfo.ActiveTopology)
}
multiPlan, err := planECDestinations(planner, metric, ecConfig)
if err != nil {
@@ -345,27 +344,21 @@ type ecPlacementPlanner struct {
candidates []*placement.DiskCandidate
candidateByKey map[string]*placement.DiskCandidate
diskStates map[string]*ecDiskState
diskTags map[string][]string
preferredTags []string
}
func newECPlacementPlanner(activeTopology *topology.ActiveTopology, preferredTags []string) *ecPlacementPlanner {
func newECPlacementPlanner(activeTopology *topology.ActiveTopology) *ecPlacementPlanner {
if activeTopology == nil {
return nil
}
disks := activeTopology.GetDisksWithEffectiveCapacity(topology.TaskTypeErasureCoding, "", 0)
candidates := diskInfosToCandidates(disks)
tagsByKey := collectDiskTags(disks)
normalizedPreferredTags := util.NormalizeTagList(preferredTags)
if len(candidates) == 0 {
return &ecPlacementPlanner{
activeTopology: activeTopology,
candidates: candidates,
candidateByKey: map[string]*placement.DiskCandidate{},
diskStates: map[string]*ecDiskState{},
diskTags: tagsByKey,
preferredTags: normalizedPreferredTags,
}
}
@@ -384,8 +377,6 @@ func newECPlacementPlanner(activeTopology *topology.ActiveTopology, preferredTag
candidates: candidates,
candidateByKey: candidateByKey,
diskStates: diskStates,
diskTags: tagsByKey,
preferredTags: normalizedPreferredTags,
}
}
@@ -406,21 +397,11 @@ func (p *ecPlacementPlanner) selectDestinations(sourceRack, sourceDC string, sha
PreferDifferentRacks: true,
}
var lastErr error
for _, candidates := range p.buildCandidateSets(shardsNeeded) {
if len(candidates) == 0 {
continue
}
result, err := placement.SelectDestinations(candidates, config)
if err == nil {
return result.SelectedDisks, nil
}
lastErr = err
result, err := placement.SelectDestinations(p.candidates, config)
if err != nil {
return nil, err
}
if lastErr == nil {
lastErr = fmt.Errorf("no EC placement candidates available")
}
return nil, lastErr
return result.SelectedDisks, nil
}
func (p *ecPlacementPlanner) applyTaskReservations(volumeSize int64, sources []topology.TaskSourceSpec, destinations []topology.TaskDestinationSpec) {
@@ -520,77 +501,6 @@ func ecDiskKey(nodeID string, diskID uint32) string {
return fmt.Sprintf("%s:%d", nodeID, diskID)
}
func collectDiskTags(disks []*topology.DiskInfo) map[string][]string {
tagMap := make(map[string][]string, len(disks))
for _, disk := range disks {
if disk == nil || disk.DiskInfo == nil {
continue
}
key := ecDiskKey(disk.NodeID, disk.DiskID)
tags := util.NormalizeTagList(disk.DiskInfo.Tags)
if len(tags) > 0 {
tagMap[key] = tags
}
}
return tagMap
}
func diskHasTag(tags []string, tag string) bool {
if tag == "" || len(tags) == 0 {
return false
}
for _, candidate := range tags {
if candidate == tag {
return true
}
}
return false
}
// buildCandidateSets builds tiered candidate sets for preferred-tag prioritized placement.
// For a planner with preferredTags, it accumulates disks matching each tag in order into
// progressively larger tiers. It emits a candidate set once a tier reaches shardsNeeded,
// then continues accumulating for subsequent tags. Finally, it falls back to the full
// p.candidates set if preferred-tag tiers are insufficient. This ensures tagged disks
// are selected first before falling back to all available candidates.
func (p *ecPlacementPlanner) buildCandidateSets(shardsNeeded int) [][]*placement.DiskCandidate {
if p == nil {
return nil
}
if len(p.preferredTags) == 0 {
return [][]*placement.DiskCandidate{p.candidates}
}
selected := make(map[string]bool, len(p.candidates))
var tier []*placement.DiskCandidate
var candidateSets [][]*placement.DiskCandidate
for _, tag := range p.preferredTags {
for _, candidate := range p.candidates {
key := ecDiskKey(candidate.NodeID, candidate.DiskID)
if selected[key] {
continue
}
if diskHasTag(p.diskTags[key], tag) {
selected[key] = true
tier = append(tier, candidate)
}
}
if shardsNeeded > 0 && len(tier) >= shardsNeeded {
candidateSets = append(candidateSets, append([]*placement.DiskCandidate(nil), tier...))
}
}
// Defensive check: selectDestinations always ensures shardsNeeded > 0 before calling
// buildCandidateSets, but this branch handles direct callers and edge cases.
if shardsNeeded <= 0 && len(tier) > 0 {
candidateSets = append(candidateSets, append([]*placement.DiskCandidate(nil), tier...))
}
if len(tier) < len(p.candidates) {
candidateSets = append(candidateSets, p.candidates)
} else if len(candidateSets) == 0 {
candidateSets = append(candidateSets, p.candidates)
}
return candidateSets
}
// planECDestinations plans the destinations for erasure coding operation
// This function implements EC destination planning logic directly in the detection phase
func planECDestinations(planner *ecPlacementPlanner, metric *types.VolumeHealthMetrics, ecConfig *Config) (*topology.MultiDestinationPlan, error) {
@@ -640,7 +550,7 @@ func planECDestinations(planner *ecPlacementPlanner, metric *types.VolumeHealthM
for _, disk := range selectedDisks {
// Get the target server address
targetAddress, err := workerutil.ResolveServerAddress(disk.NodeID, planner.activeTopology)
targetAddress, err := util.ResolveServerAddress(disk.NodeID, planner.activeTopology)
if err != nil {
return nil, fmt.Errorf("failed to resolve address for target server %s: %v", disk.NodeID, err)
}
@@ -744,7 +654,7 @@ func convertTaskSourcesToProtobuf(sources []topology.TaskSourceSpec, volumeID ui
var protobufSources []*worker_pb.TaskSource
for _, source := range sources {
serverAddress, err := workerutil.ResolveServerAddress(source.ServerID, activeTopology)
serverAddress, err := util.ResolveServerAddress(source.ServerID, activeTopology)
if err != nil {
return nil, fmt.Errorf("failed to resolve address for source server %s: %v", source.ServerID, err)
}
@@ -17,7 +17,7 @@ import (
func TestECPlacementPlannerApplyReservations(t *testing.T) {
activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 10, 0)
planner := newECPlacementPlanner(activeTopology, nil)
planner := newECPlacementPlanner(activeTopology)
require.NotNil(t, planner)
key := ecDiskKey("10.0.0.1:8080", 0)
@@ -47,7 +47,7 @@ func TestECPlacementPlannerApplyReservations(t *testing.T) {
func TestPlanECDestinationsUsesPlanner(t *testing.T) {
activeTopology := buildActiveTopology(t, 7, []string{"hdd", "ssd"}, 100, 0)
planner := newECPlacementPlanner(activeTopology, nil)
planner := newECPlacementPlanner(activeTopology)
require.NotNil(t, planner)
metric := &types.VolumeHealthMetrics{
@@ -63,70 +63,6 @@ func TestPlanECDestinationsUsesPlanner(t *testing.T) {
assert.Equal(t, erasure_coding.TotalShardsCount, len(plan.Plans))
}
func TestECPlacementPlannerPrefersTaggedDisks(t *testing.T) {
activeTopology := buildActiveTopology(t, 3, []string{"hdd"}, 10, 0)
topo := activeTopology.GetTopologyInfo()
for _, dc := range topo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for k, node := range rack.DataNodeInfos {
for diskType := range node.DiskInfos {
if k < 2 {
node.DiskInfos[diskType].Tags = []string{"fast"}
} else {
node.DiskInfos[diskType].Tags = []string{"slow"}
}
}
}
}
}
require.NoError(t, activeTopology.UpdateTopology(topo))
planner := newECPlacementPlanner(activeTopology, []string{"fast"})
require.NotNil(t, planner)
selected, err := planner.selectDestinations("", "", 2)
require.NoError(t, err)
require.Len(t, selected, 2)
for _, candidate := range selected {
key := ecDiskKey(candidate.NodeID, candidate.DiskID)
assert.True(t, diskHasTag(planner.diskTags[key], "fast"))
}
}
func TestECPlacementPlannerFallsBackWhenTagsInsufficient(t *testing.T) {
activeTopology := buildActiveTopology(t, 3, []string{"hdd"}, 10, 0)
topo := activeTopology.GetTopologyInfo()
for _, dc := range topo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for i, node := range rack.DataNodeInfos {
for diskType := range node.DiskInfos {
if i == 0 {
node.DiskInfos[diskType].Tags = []string{"fast"}
}
}
}
}
}
require.NoError(t, activeTopology.UpdateTopology(topo))
planner := newECPlacementPlanner(activeTopology, []string{"fast"})
require.NotNil(t, planner)
selected, err := planner.selectDestinations("", "", 3)
require.NoError(t, err)
require.Len(t, selected, 3)
taggedCount := 0
for _, candidate := range selected {
key := ecDiskKey(candidate.NodeID, candidate.DiskID)
if diskHasTag(planner.diskTags[key], "fast") {
taggedCount++
}
}
assert.Less(t, taggedCount, len(selected))
}
func TestDetectionContextCancellation(t *testing.T) {
activeTopology := buildActiveTopology(t, 5, []string{"hdd", "ssd"}, 50, 0)
clusterInfo := &types.ClusterInfo{ActiveTopology: activeTopology}
@@ -152,7 +88,7 @@ func TestDetectionMaxResultsHonorsLimit(t *testing.T) {
func TestPlanECDestinationsFailsWithInsufficientCapacity(t *testing.T) {
activeTopology := buildActiveTopology(t, 1, []string{"hdd"}, 1, 1)
planner := newECPlacementPlanner(activeTopology, nil)
planner := newECPlacementPlanner(activeTopology)
require.NotNil(t, planner)
metric := &types.VolumeHealthMetrics{
+11 -5
View File
@@ -34,11 +34,12 @@ type ErasureCodingTask struct {
grpcDialOption grpc.DialOption
// EC parameters
dataShards int32
parityShards int32
targets []*worker_pb.TaskTarget // Unified targets for EC shards
sources []*worker_pb.TaskSource // Unified sources for cleanup
shardAssignment map[string][]string // destination -> assigned shard types
dataShards int32
parityShards int32
targets []*worker_pb.TaskTarget // Unified targets for EC shards
sources []*worker_pb.TaskSource // Unified sources for cleanup
shardAssignment map[string][]string // destination -> assigned shard types
allowSourceDelete bool
}
// NewErasureCodingTask creates a new unified EC task instance
@@ -179,6 +180,7 @@ func (t *ErasureCodingTask) Execute(ctx context.Context, params *worker_pb.TaskP
if err := t.mountEcShards(); err != nil {
return fmt.Errorf("failed to mount EC shards: %v", err)
}
t.allowSourceDelete = true
// Step 6: Delete original volume
t.ReportProgressWithStage(90.0, "Deleting original volume")
@@ -469,6 +471,10 @@ func (t *ErasureCodingTask) mountEcShards() error {
// deleteOriginalVolume deletes the original volume and all its replicas from all servers
func (t *ErasureCodingTask) deleteOriginalVolume() error {
if !t.allowSourceDelete {
return fmt.Errorf("source deletion disabled: EC shards not fully distributed and mounted")
}
// Get replicas from task parameters (set during detection)
replicas := t.getReplicas()