Compare commits

..
Author SHA1 Message Date
Chris LuandCopilot 8500a3bc56 ci: remove s3tests skip - test will be fixed in s3-tests repo
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 03:23:36 -08:00
Chris LuandCopilot 678c52e74f ci: skip s3tests test expecting incorrect delete behavior
Skip test_object_lock_delete_object_with_retention_and_marker in s3tests because it expects SeaweedFS incorrect behavior (allowing delete under COMPLIANCE retention). SeaweedFS now correctly implements AWS S3 behavior: delete of objects under COMPLIANCE retention returns AccessDenied and does not create a delete marker.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 03:20:05 -08:00
Chris LuandCopilot 8fac0cdab8 test: fix retention tests to expect correct AWS S3 behavior
Tests were expecting SeaweedFS old (incorrect) behavior where DeleteObject succeeded under COMPLIANCE retention. Updated to expect correct AWS S3 behavior: simple DELETE (without versionId) is blocked by active COMPLIANCE/legal-hold, returning AccessDenied instead of creating a delete marker.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 02:57:31 -08:00
Chris LuandCopilot 09bfb28db9 s3api: simplify objectLockVersionToCheckForDelete to single return value
Since all branches now return true (fail-closed), eliminate the boolean return value and the dead-code if guards. The function now clearly indicates that object lock checks always execute, with no opt-out path.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 02:08:32 -08:00
Chris LuandCopilot 3bd990d2ea s3api: address object lock delete review comments
- Remove versioningConfigured guard: object lock protections must apply to all buckets
- Combine identical switch cases for clarity
- Change default case to fail-closed (true) for safety
- Add test case for suspended versioning with specific versionId
- Update test expectations to reflect fail-closed default

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 02:00:30 -08:00
Chris LuandCopilot 5e040ba03d s3api: enforce object lock protections on delete (COMPLIANCE/GOVERNANCE)
Enforce object lock protections before creating delete markers; active COMPLIANCE retention denies deletes and active GOVERNANCE retention denies deletes unless bypass is authorized. Includes unit tests.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 01:52:08 -08:00
112 changed files with 367 additions and 15336 deletions
+1 -1
View File
@@ -111,7 +111,7 @@ jobs:
# Wait for S3 API
for i in {1..30}; do
if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
if curl -s http://localhost:8333/ > /dev/null 2>&1; then
echo "✓ S3 API is ready"
break
fi
-2
View File
@@ -461,8 +461,6 @@ jobs:
export S3_ENDPOINT="http://localhost:8006"
export S3_ACCESS_KEY="0555b35654ad1656d804"
export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
# Run the specific test that is equivalent to AWS S3 tagging behavior
make test-with-server || {
+1 -1
View File
@@ -97,7 +97,7 @@ jobs:
# Verify service accessibility
echo "=== Verifying Service Accessibility ==="
curl -f http://localhost:8080/realms/master
curl -s http://localhost:8333/healthz
curl -s http://localhost:8333
echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
# Run Keycloak-specific tests
@@ -35,7 +35,7 @@ jobs:
set -e
mkdir -p /tmp/data
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
- name: Setup Caddy
run: |
@@ -54,7 +54,7 @@ jobs:
- name: Start Caddy
run: |
./caddy start
until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
- name: Create Bucket
run: |
+5 -5
View File
@@ -46,7 +46,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/klauspost/compress v1.18.4
github.com/klauspost/compress v1.18.3
github.com/klauspost/reedsolomon v1.13.0
github.com/kurin/blazer v0.5.3
github.com/linxGnu/grocksdb v1.10.7
@@ -130,7 +130,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
github.com/cognusion/imaging v1.0.2
github.com/fluent/fluent-logger-golang v1.10.1
github.com/getsentry/sentry-go v0.42.0
github.com/getsentry/sentry-go v0.40.0
github.com/gin-contrib/sessions v1.0.4
github.com/gin-gonic/gin v1.11.0
github.com/go-ldap/ldap/v3 v3.4.12
@@ -142,7 +142,7 @@ require (
github.com/jhump/protoreflect v1.18.0
github.com/lib/pq v1.11.1
github.com/linkedin/goavro/v2 v2.14.1
github.com/mattn/go-sqlite3 v1.14.34
github.com/mattn/go-sqlite3 v1.14.33
github.com/minio/crc64nvme v1.1.1
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/parquet-go/parquet-go v0.26.4
@@ -150,7 +150,7 @@ require (
github.com/rabbitmq/amqp091-go v1.10.0
github.com/rclone/rclone v1.72.1
github.com/rdleal/intervalst v1.5.0
github.com/redis/go-redis/v9 v9.18.0
github.com/redis/go-redis/v9 v9.17.2
github.com/schollz/progressbar/v3 v3.19.0
github.com/seaweedfs/go-fuse/v2 v2.9.1
github.com/shirou/gopsutil/v4 v4.26.1
@@ -297,7 +297,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
github.com/Azure/go-ntlmssp v0.1.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
+10 -10
View File
@@ -569,8 +569,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -1053,8 +1053,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo=
github.com/getsentry/sentry-go v0.40.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
@@ -1487,8 +1487,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -1564,8 +1564,8 @@ github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhg
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -1814,8 +1814,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.13"
appVersion: "4.12"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.0.413
version: 4.0.412
+6
View File
@@ -19,6 +19,9 @@ services:
start_period: 10s
seaweedfs-volume:
build:
context: ../../../docker
dockerfile: Dockerfile.local
image: seaweedfs:local
container_name: seaweedfs-spark-volume
ports:
@@ -40,6 +43,9 @@ services:
start_period: 10s
seaweedfs-filer:
build:
context: ../../../docker
dockerfile: Dockerfile.local
image: seaweedfs:local
container_name: seaweedfs-spark-filer
ports:
+1 -1
View File
@@ -116,7 +116,7 @@
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.11.5</version>
<version>1.11.4</version>
</dependency>
<!-- Apache ZooKeeper - Fix CVEs -->
+4 -7
View File
@@ -12,7 +12,6 @@ FILER_PORT := 8888
TEST_TIMEOUT := 10m
TEST_PATTERN := TestCORS
SERVER_DIR := test-mini-data
S3_CONFIG := s3_test_config.json
# Default target
help:
@@ -81,15 +80,13 @@ start-server: check-deps
@echo "🔍 DEBUG: Creating volume directory..."
@mkdir -p $(SERVER_DIR)
@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
@echo "🔍 DEBUG: Command: AWS_ACCESS_KEY_ID=some_access_key1 AWS_SECRET_ACCESS_KEY=some_secret_key1 $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
@env AWS_ACCESS_KEY_ID=some_access_key1 \
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
$(WEED_BINARY) mini \
@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
@$(WEED_BINARY) mini \
-dir=$(SERVER_DIR) \
-s3.port=$(S3_PORT) \
-s3.config=$(S3_CONFIG) \
> weed-test.log 2>&1 & \
echo $$! > weed-server.pid
echo $$! > weed-test.pid
@echo "Waiting for S3 server to be ready..."
@for i in $$(seq 1 30); do \
@@ -100,7 +97,7 @@ start-server: check-deps
sleep 1; \
done; \
echo "S3 server failed to start"; \
exit 1
exit 1 > weed-server.pid
@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
@echo "🔍 DEBUG: Checking if PID is still running..."
@sleep 2
-27
View File
@@ -1,27 +0,0 @@
{
"identities": [
{
"name": "anonymous",
"actions": [
"Read",
"List"
]
},
{
"name": "admin",
"credentials": [
{
"accessKey": "some_access_key1",
"secretKey": "some_secret_key1"
}
],
"actions": [
"Admin",
"Read",
"List",
"Tagging",
"Write"
]
}
]
}
+5 -2
View File
@@ -1,6 +1,7 @@
package example
import (
"os"
"testing"
"time"
@@ -21,8 +22,10 @@ func TestIAMOperations(t *testing.T) {
// Set credentials before starting cluster
accessKey := "testkey123"
secretKey := "testsecret456"
t.Setenv("AWS_ACCESS_KEY_ID", accessKey)
t.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
os.Setenv("AWS_ACCESS_KEY_ID", accessKey)
os.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
defer os.Unsetenv("AWS_ACCESS_KEY_ID")
defer os.Unsetenv("AWS_SECRET_ACCESS_KEY")
// Create and start test cluster
cluster, err := startMiniCluster(t)
-8
View File
@@ -146,14 +146,6 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
return nil, fmt.Errorf("failed to create security.toml: %v", err)
}
// Set environment variables for admin credentials safely for this test
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
}
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
}
// Start weed mini in a goroutine by calling the command directly
cluster.wg.Add(1)
go func() {
-8
View File
@@ -216,14 +216,6 @@ enabled = true
err = os.WriteFile(credentialToml, []byte(credentialConfig), 0644)
require.NoError(t, err)
// Set environment variables for admin credentials safely for this test
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
}
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
}
cluster.wg.Add(1)
go func() {
defer cluster.wg.Done()
@@ -77,14 +77,14 @@ func TestObjectLockValidation(t *testing.T) {
require.NoError(t, err, "Setting Object Lock retention should succeed")
t.Log(" Object Lock retention applied successfully")
// Verify retention allows simple DELETE (creates delete marker) but blocks version deletion
// AWS S3 behavior: Simple DELETE (without version ID) is ALWAYS allowed and creates delete marker
// Verify retention blocks simple DELETE (COMPLIANCE mode is strict WORM)
// AWS S3 behavior: Simple DELETE (without version ID) is blocked by COMPLIANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker (AWS S3 behavior)")
t.Log(" Simple DELETE succeeded (creates delete marker - correct AWS behavior)")
require.Error(t, err, "Simple DELETE should be blocked by COMPLIANCE retention (AWS S3 behavior)")
t.Log(" Simple DELETE correctly blocked by COMPLIANCE retention")
// Now verify that DELETE with version ID is properly blocked by retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+7 -7
View File
@@ -329,12 +329,12 @@ func TestRetentionModeCompliance(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, types.ObjectLockRetentionModeCompliance, retentionResp.Retention.Mode)
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
// Try simple DELETE - should fail for COMPLIANCE mode (strict WORM)
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
require.Error(t, err, "Simple DELETE should be blocked by COMPLIANCE retention")
// Try DELETE with version ID - should fail for COMPLIANCE mode
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
@@ -388,12 +388,12 @@ func TestLegalHoldWorkflow(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, legalHoldResp.LegalHold.Status)
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
// Try simple DELETE - should fail due to legal hold
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
require.Error(t, err, "Simple DELETE should be blocked by legal hold")
// Try DELETE with version ID - should fail due to legal hold
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
@@ -591,12 +591,12 @@ func TestRetentionAndLegalHoldCombination(t *testing.T) {
})
require.NoError(t, err)
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
// Try simple DELETE - should fail due to legal hold + COMPLIANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
require.Error(t, err, "Simple DELETE should be blocked by legal hold")
// Try DELETE with version ID and bypass - should still fail due to legal hold
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
@@ -607,7 +607,7 @@ func TestRetentionAndLegalHoldCombination(t *testing.T) {
})
require.Error(t, err, "Legal hold should prevent deletion even with governance bypass")
// Remove legal hold (must specify version ID since latest version is now delete marker)
// Remove legal hold (must specify version ID)
_, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
@@ -42,12 +42,12 @@ func TestWORMRetentionIntegration(t *testing.T) {
})
require.NoError(t, err)
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
// Try simple DELETE - should fail due to GOVERNANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
require.Error(t, err, "Simple DELETE should be blocked by GOVERNANCE retention")
// Try DELETE with version ID - should fail due to GOVERNANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
@@ -325,12 +325,12 @@ func TestRetentionWithMultipartUpload(t *testing.T) {
})
require.NoError(t, err)
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
// Try simple DELETE - should fail due to GOVERNANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
require.Error(t, err, "Simple DELETE should be blocked by GOVERNANCE retention")
// Try DELETE with version ID - should fail due to GOVERNANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
@@ -511,10 +511,8 @@ func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
t.Logf("Create table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("Failed to create table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
}
t.Logf("Created table bucket %s", bucketName)
+2 -66
View File
@@ -56,6 +56,8 @@ func TestPyIcebergRestCatalog(t *testing.T) {
cmd := exec.Command("docker", "run", "--rm",
"--add-host", "host.docker.internal:host-gateway",
"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", "test"),
"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", "test"),
"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
"-v", fmt.Sprintf("%s:/app:ro", testDir),
"iceberg-rest-test",
@@ -76,69 +78,3 @@ func TestPyIcebergRestCatalog(t *testing.T) {
t.Errorf("PyIceberg test failed: %v", err)
}
}
// TestPyIcebergRestCatalogAuthenticated tests the Iceberg REST Catalog using PyIceberg with authentication.
// This test uses the default admin credentials that SeaweedFS creates on startup.
func TestPyIcebergRestCatalogAuthenticated(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
env := NewTestEnvironment(t)
defer env.Cleanup(t)
if !env.dockerAvailable {
t.Skip("Docker not available, skipping PyIceberg integration test")
}
// Use default admin credentials
testAccessKey := "admin"
testSecretKey := "admin"
// Start SeaweedFS (it will use default admin credentials from environment if set)
env.StartSeaweedFS(t)
// Create the test bucket first (using unauthenticated request, which works with DefaultAllow)
bucketName := "pyiceberg-auth-test"
createTableBucket(t, env, bucketName)
// Build the test working directory path
testDir := filepath.Join(env.seaweedDir, "test", "s3tables", "catalog")
// Run PyIceberg test using Docker with authentication
catalogURL := fmt.Sprintf("http://host.docker.internal:%d", env.icebergPort)
s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
warehouse := fmt.Sprintf("s3://%s/", bucketName)
// Build the test image first for faster repeated runs
buildCmd := exec.Command("docker", "build", "-t", "iceberg-rest-test", "-f", "Dockerfile.pyiceberg", ".")
buildCmd.Dir = testDir
if out, err := buildCmd.CombinedOutput(); err != nil {
t.Fatalf("Failed to build test image: %v\n%s", err, string(out))
}
cmd := exec.Command("docker", "run", "--rm",
"--add-host", "host.docker.internal:host-gateway",
"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
"-v", fmt.Sprintf("%s:/app:ro", testDir),
"iceberg-rest-test",
"python3", "/app/test_rest_catalog_auth.py",
"--catalog-url", catalogURL,
"--warehouse", warehouse,
"--prefix", bucketName,
"--access-key", testAccessKey,
"--secret-key", testSecretKey,
)
cmd.Dir = testDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
t.Logf("Running PyIceberg REST catalog test with authentication...")
t.Logf(" Catalog URL: %s", catalogURL)
t.Logf(" Warehouse: %s", warehouse)
t.Logf(" Access Key: %s", testAccessKey)
if err := cmd.Run(); err != nil {
t.Errorf("PyIceberg authenticated test failed: %v", err)
}
}
@@ -1,36 +0,0 @@
package catalog
import (
"fmt"
"io"
"net/http"
"testing"
)
// verifyTableBucketMetadata verifies that a table bucket was created with proper metadata
func verifyTableBucketMetadata(t *testing.T, env *TestEnvironment, bucketName string) {
t.Helper()
// Use S3Tables REST API to get the bucket
endpoint := fmt.Sprintf("http://localhost:%d/buckets/%s", env.s3Port, bucketName)
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
if err != nil {
t.Fatalf("Failed to create request: %v", err)
}
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Failed to get table bucket %s: %v", bucketName, err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
t.Logf("Get table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
if resp.StatusCode != http.StatusOK {
t.Fatalf("Failed to get table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
}
t.Logf("Verified table bucket %s exists with metadata", bucketName)
}
@@ -201,7 +201,6 @@ def main():
"uri": args.catalog_url,
"warehouse": args.warehouse,
"prefix": args.prefix,
"s3.anonymous": "true", # Disable AWS request signing for unauthenticated access
}
)
print(f"Successfully connected to catalog on attempt {attempt + 1}")
@@ -1,223 +0,0 @@
#!/usr/bin/env python3
"""
Iceberg REST Catalog Compatibility Test for SeaweedFS (Authenticated)
This script tests the Iceberg REST Catalog API compatibility with authentication.
Usage:
python3 test_rest_catalog_auth.py --catalog-url http://localhost:8182 \\
--access-key admin --secret-key admin
Requirements:
pip install pyiceberg[s3fs]
"""
import argparse
import sys
from pyiceberg.catalog import load_catalog
from pyiceberg.schema import Schema
from pyiceberg.types import (
IntegerType,
LongType,
StringType,
NestedField,
)
from pyiceberg.exceptions import (
NamespaceAlreadyExistsError,
NoSuchNamespaceError,
TableAlreadyExistsError,
NoSuchTableError,
)
def test_config_endpoint(catalog):
"""Test that the catalog config endpoint returns valid configuration."""
print("Testing /v1/config endpoint...")
# The catalog is already loaded which means config endpoint worked
print(" /v1/config endpoint working")
return True
def test_namespace_operations(catalog, prefix):
"""Test namespace CRUD operations."""
print("Testing namespace operations...")
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
# List initial namespaces
namespaces = catalog.list_namespaces()
print(f" Initial namespaces: {namespaces}")
# Create namespace
try:
catalog.create_namespace(namespace)
print(f" Created namespace: {namespace}")
except NamespaceAlreadyExistsError:
print(f" ! Namespace already exists: {namespace}")
# List namespaces (should include our new one)
namespaces = catalog.list_namespaces()
if namespace in namespaces:
print(" Namespace appears in list")
else:
print(f" Namespace not found in list: {namespaces}")
return False
# Get namespace properties
try:
props = catalog.load_namespace_properties(namespace)
print(f" Loaded namespace properties: {props}")
except NoSuchNamespaceError:
print(f" Failed to load namespace properties")
return False
return True
def test_table_operations(catalog, prefix):
"""Test table CRUD operations."""
print("Testing table operations...")
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
table_name = "auth_test_table"
table_id = namespace + (table_name,)
# Define a simple schema
schema = Schema(
NestedField(field_id=1, name="id", field_type=LongType(), required=True),
NestedField(field_id=2, name="name", field_type=StringType(), required=False),
NestedField(field_id=3, name="age", field_type=IntegerType(), required=False),
)
# Create table
try:
table = catalog.create_table(
identifier=table_id,
schema=schema,
)
print(f" Created table: {table_id}")
except TableAlreadyExistsError:
print(f" ! Table already exists: {table_id}")
_ = catalog.load_table(table_id)
# List tables
tables = catalog.list_tables(namespace)
if table_name in [t[1] for t in tables]:
print(" Table appears in list")
else:
print(f" Table not found in list: {tables}")
return False
# Load table
try:
loaded_table = catalog.load_table(table_id)
print(f" Loaded table: {loaded_table.name()}")
print(f" Schema: {loaded_table.schema()}")
print(f" Location: {loaded_table.location()}")
except NoSuchTableError:
print(f" Failed to load table")
return False
return True
def test_cleanup(catalog, prefix):
"""Test table and namespace deletion."""
print("Testing cleanup operations...")
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
table_id = namespace + ("auth_test_table",)
# Drop table
try:
catalog.drop_table(table_id)
print(f" Dropped table: {table_id}")
except NoSuchTableError:
print(f" ! Table already deleted: {table_id}")
# Drop namespace
try:
catalog.drop_namespace(namespace)
print(f" Dropped namespace: {namespace}")
except NoSuchNamespaceError:
print(f" ! Namespace already deleted: {namespace}")
except Exception as e:
print(f" ? Namespace drop error (may be expected): {e}")
return True
def main():
parser = argparse.ArgumentParser(description="Test Iceberg REST Catalog with authentication")
parser.add_argument("--catalog-url", required=True, help="Iceberg REST Catalog URL")
parser.add_argument("--warehouse", default="s3://iceberg-test/", help="Warehouse location")
parser.add_argument("--prefix", required=True, help="Table bucket prefix")
parser.add_argument("--access-key", required=True, help="AWS Access Key ID")
parser.add_argument("--secret-key", required=True, help="AWS Secret Access Key")
parser.add_argument("--skip-cleanup", action="store_true", help="Skip cleanup at the end")
args = parser.parse_args()
print(f"Connecting to Iceberg REST Catalog at: {args.catalog_url}")
print(f"Warehouse: {args.warehouse}")
print(f"Prefix: {args.prefix}")
print(f"Using authenticated access with key: {args.access_key}")
print()
# Load the REST catalog with authentication
import time
max_retries = 10
catalog = None
for attempt in range(max_retries):
try:
catalog = load_catalog(
"rest",
**{
"type": "rest",
"uri": args.catalog_url,
"warehouse": args.warehouse,
"prefix": args.prefix,
"s3.access-key-id": args.access_key,
"s3.secret-access-key": args.secret_key,
}
)
print(f"Successfully connected to catalog on attempt {attempt + 1}")
break
except Exception as e:
if attempt < max_retries - 1:
print(f" Attempt {attempt + 1} failed, retrying in 2s... ({e})")
time.sleep(2)
else:
print(f" All {max_retries} attempts failed.")
raise e
# Run tests
tests = [
("Config Endpoint", lambda: test_config_endpoint(catalog)),
("Namespace Operations", lambda: test_namespace_operations(catalog, args.prefix)),
("Table Operations", lambda: test_table_operations(catalog, args.prefix)),
]
if not args.skip_cleanup:
tests.append(("Cleanup", lambda: test_cleanup(catalog, args.prefix)))
passed = 0
failed = 0
for name, test_fn in tests:
print(f"\n{'='*50}")
try:
if test_fn():
passed += 1
print(f"PASSED: {name}")
else:
failed += 1
print(f"FAILED: {name}")
except Exception as e:
failed += 1
print(f"ERROR in {name}: {e}")
print(f"\n{'='*50}")
print(f"Results: {passed} passed, {failed} failed")
return 0 if failed == 0 else 1
if __name__ == "__main__":
sys.exit(main())
@@ -556,14 +556,6 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
return nil, fmt.Errorf("failed to create security.toml: %v", err)
}
// Set environment variables for admin credentials safely for this test
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
}
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
}
// Start weed mini in a goroutine by calling the command directly
cluster.wg.Add(1)
go func() {
-41
View File
@@ -4,15 +4,12 @@ import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/credential"
"github.com/seaweedfs/seaweedfs/weed/glog"
@@ -118,9 +115,6 @@ type AdminServer struct {
s3TablesManager *s3tables.Manager
icebergPort int
// Plugin system manager
pluginManager interface{}
}
// Type definitions moved to types.go
@@ -232,9 +226,6 @@ func NewAdminServer(masters string, templateFS http.FileSystem, dataDir string,
}()
}
// Initialize plugin manager
server.initPluginManager(dataDir)
return server
}
@@ -255,38 +246,6 @@ func (s *AdminServer) GetCredentialManager() *credential.CredentialManager {
return s.credentialManager
}
// initPluginManager initializes the plugin manager
func (s *AdminServer) initPluginManager(dataDir string) {
// Create plugin configuration directory if it doesn't exist
pluginConfigDir := filepath.Join(dataDir, "plugins")
if err := os.MkdirAll(pluginConfigDir, 0755); err != nil {
glog.Warningf("Failed to create plugin config directory: %v", err)
return
}
// Create plugin manager with default configuration
config := plugin.DefaultManagerConfig(pluginConfigDir)
pm, err := plugin.NewManager(config)
if err != nil {
glog.Warningf("Failed to initialize plugin manager: %v", err)
return
}
// Store the plugin manager
s.pluginManager = pm
glog.Infof("Plugin manager initialized successfully")
}
// GetPluginManager returns the plugin manager
func (s *AdminServer) GetPluginManager() interface{} {
return s.pluginManager
}
// SetPluginManager sets the plugin manager
func (s *AdminServer) SetPluginManager(pm interface{}) {
s.pluginManager = pm
}
// Filer discovery methods moved to client_management.go
// Client management methods moved to client_management.go
-15
View File
@@ -9,10 +9,8 @@ import (
"time"
"github.com/seaweedfs/seaweedfs/weed/admin/maintenance"
"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -96,19 +94,6 @@ func (s *WorkerGrpcServer) StartWithTLS(port int) error {
worker_pb.RegisterWorkerServiceServer(grpcServer, s)
// Register plugin service if plugin manager is available
if s.adminServer.GetPluginManager() != nil {
// Cast the interface{} to *plugin.Manager
if pm, ok := s.adminServer.GetPluginManager().(*plugin.Manager); ok {
if pluginGrpcServer := pm.GetGRPCServer(); pluginGrpcServer != nil {
plugin_pb.RegisterPluginServiceServer(grpcServer, pluginGrpcServer)
plugin_pb.RegisterAdminQueryServiceServer(grpcServer, pluginGrpcServer)
plugin_pb.RegisterAdminCommandServiceServer(grpcServer, pluginGrpcServer)
glog.Infof("Registered plugin services on worker gRPC server")
}
}
}
s.grpcServer = grpcServer
s.listener = listener
s.running = true
-141
View File
@@ -3,13 +3,11 @@ package handlers
import (
"net/http"
"net/url"
"sort"
"time"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/seaweedfs/seaweedfs/weed/admin/dash"
"github.com/seaweedfs/seaweedfs/weed/admin/plugin"
"github.com/seaweedfs/seaweedfs/weed/admin/view/app"
"github.com/seaweedfs/seaweedfs/weed/admin/view/layout"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
@@ -28,7 +26,6 @@ type AdminHandlers struct {
maintenanceHandlers *MaintenanceHandlers
mqHandlers *MessageQueueHandlers
serviceAccountHandlers *ServiceAccountHandlers
pluginHandlers *PluginHandlers
}
// NewAdminHandlers creates a new instance of AdminHandlers
@@ -41,14 +38,6 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers {
maintenanceHandlers := NewMaintenanceHandlers(adminServer)
mqHandlers := NewMessageQueueHandlers(adminServer)
serviceAccountHandlers := NewServiceAccountHandlers(adminServer)
// Get plugin manager from admin server (may be nil)
var pluginMgr interface{}
if pm := adminServer.GetPluginManager(); pm != nil {
pluginMgr = pm
}
pluginHandlers := NewPluginHandlers(adminServer, pluginMgr)
return &AdminHandlers{
adminServer: adminServer,
authHandlers: authHandlers,
@@ -59,7 +48,6 @@ func NewAdminHandlers(adminServer *dash.AdminServer) *AdminHandlers {
maintenanceHandlers: maintenanceHandlers,
mqHandlers: mqHandlers,
serviceAccountHandlers: serviceAccountHandlers,
pluginHandlers: pluginHandlers,
}
}
@@ -131,11 +119,6 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
protected.GET("/mq/topics", h.mqHandlers.ShowTopics)
protected.GET("/mq/topics/:namespace/:topic", h.mqHandlers.ShowTopicDetails)
// Plugin management routes
protected.GET("/plugins", h.ShowPlugins)
protected.GET("/plugins/jobs/:jobType", h.ShowPluginJobs)
protected.GET("/plugins/config/:jobType", h.ShowPluginConfig)
// Maintenance system routes
protected.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue)
protected.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers)
@@ -267,19 +250,6 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
mqApi.POST("/topics/retention/update", dash.RequireWriteAccess(), h.mqHandlers.UpdateTopicRetentionAPI)
mqApi.POST("/retention/purge", dash.RequireWriteAccess(), h.adminServer.TriggerTopicRetentionPurgeAPI)
}
// Plugin API routes
pluginApi := api.Group("/plugin")
{
pluginApi.GET("/list", h.pluginHandlers.ListPluginsAPI)
pluginApi.GET("/jobs/by-type/:type", h.pluginHandlers.ListJobsAPI)
pluginApi.GET("/config/:type", h.pluginHandlers.GetConfigAPI)
pluginApi.POST("/config/:type/apply", dash.RequireWriteAccess(), h.pluginHandlers.SaveConfigAPI)
pluginApi.GET("/detection/history/:type", h.pluginHandlers.GetDetectionHistoryAPI)
pluginApi.GET("/execution/history/:type", h.pluginHandlers.GetExecutionHistoryAPI)
pluginApi.POST("/trigger-detection/:type", dash.RequireWriteAccess(), h.pluginHandlers.TriggerDetectionAPI)
pluginApi.POST("/cancel-job/:id", dash.RequireWriteAccess(), h.pluginHandlers.CancelJobAPI)
}
}
} else {
// No authentication required - all routes are public
@@ -322,11 +292,6 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
r.GET("/mq/topics", h.mqHandlers.ShowTopics)
r.GET("/mq/topics/:namespace/:topic", h.mqHandlers.ShowTopicDetails)
// Plugin management routes
r.GET("/plugins", h.ShowPlugins)
r.GET("/plugins/jobs/:jobType", h.ShowPluginJobs)
r.GET("/plugins/config/:jobType", h.ShowPluginConfig)
// Maintenance system routes
r.GET("/maintenance", h.maintenanceHandlers.ShowMaintenanceQueue)
r.GET("/maintenance/workers", h.maintenanceHandlers.ShowMaintenanceWorkers)
@@ -457,19 +422,6 @@ func (h *AdminHandlers) SetupRoutes(r *gin.Engine, authRequired bool, adminUser,
mqApi.POST("/topics/retention/update", h.mqHandlers.UpdateTopicRetentionAPI)
mqApi.POST("/retention/purge", h.adminServer.TriggerTopicRetentionPurgeAPI)
}
// Plugin API routes
pluginApi := api.Group("/plugin")
{
pluginApi.GET("/list", h.pluginHandlers.ListPluginsAPI)
pluginApi.GET("/jobs/by-type/:type", h.pluginHandlers.ListJobsAPI)
pluginApi.GET("/config/:type", h.pluginHandlers.GetConfigAPI)
pluginApi.POST("/config/:type/apply", h.pluginHandlers.SaveConfigAPI)
pluginApi.GET("/detection/history/:type", h.pluginHandlers.GetDetectionHistoryAPI)
pluginApi.GET("/execution/history/:type", h.pluginHandlers.GetExecutionHistoryAPI)
pluginApi.POST("/trigger-detection/:type", h.pluginHandlers.TriggerDetectionAPI)
pluginApi.POST("/cancel-job/:id", h.pluginHandlers.CancelJobAPI)
}
}
}
}
@@ -718,97 +670,4 @@ func (h *AdminHandlers) getAdminData(c *gin.Context) dash.AdminData {
return adminData
}
// ShowPlugins displays the plugins overview page
func (h *AdminHandlers) ShowPlugins(c *gin.Context) {
plugins := []map[string]interface{}{}
jobTypes := make(map[string]interface{})
// Get plugin manager from server
if pm := h.adminServer.GetPluginManager(); pm != nil {
// Cast to *plugin.Manager
if pluginMgr, ok := pm.(*plugin.Manager); ok {
// Get list of connected plugins
connectedPlugins := pluginMgr.ListPlugins(false)
for _, p := range connectedPlugins {
plugins = append(plugins, map[string]interface{}{
"id": p.ID,
"name": p.Name,
"version": p.Version,
"status": p.Status,
"capabilities": p.Capabilities,
"activeJobs": p.ActiveJobs,
"completedJobs": p.CompletedJobs,
"failedJobs": p.FailedJobs,
"connectedAt": p.ConnectedAt,
"lastHeartbeat": p.LastHeartbeat,
})
// Build job types map
for _, cap := range p.Capabilities {
if _, exists := jobTypes[cap]; !exists {
jobTypes[cap] = map[string]interface{}{
"type": cap,
"description": cap,
"pluginCount": 0,
}
}
// Increment plugin count for this capability
if capData, ok := jobTypes[cap].(map[string]interface{}); ok {
capData["pluginCount"] = capData["pluginCount"].(int) + 1
}
}
}
}
}
// Sort plugins by ID
sort.Slice(plugins, func(i, j int) bool {
return plugins[i]["id"].(string) < plugins[j]["id"].(string)
})
component := app.PluginsOverview(app.PluginsPageData{
Plugins: plugins,
JobTypes: jobTypes,
})
htmlContent := layout.Layout(c, component)
htmlContent.Render(c.Request.Context(), c.Writer)
}
// ShowPluginJobs displays the job monitoring page for a specific type
func (h *AdminHandlers) ShowPluginJobs(c *gin.Context) {
jobType := c.Param("jobType")
jobs := []interface{}{}
stateFilter := c.Query("state")
component := app.PluginJobsMonitoring(app.PluginJobsPageData{
JobType: jobType,
Jobs: jobs,
StateFilter: stateFilter,
})
htmlContent := layout.Layout(c, component)
htmlContent.Render(c.Request.Context(), c.Writer)
}
// ShowPluginConfig displays the configuration page for a job type
func (h *AdminHandlers) ShowPluginConfig(c *gin.Context) {
jobType := c.Param("jobType")
activeTab := c.Query("tab")
if activeTab == "" {
activeTab = "config"
}
component := app.PluginConfiguration(app.PluginConfigPageData{
JobType: jobType,
Config: app.JobTypeConfig{},
DetectionHistory: []interface{}{},
ExecutionHistory: []interface{}{},
ActiveTab: activeTab,
})
htmlContent := layout.Layout(c, component)
htmlContent.Render(c.Request.Context(), c.Writer)
}
// Helper functions
-95
View File
@@ -1,95 +0,0 @@
package handlers
import (
"net/http"
"github.com/gin-gonic/gin"
)
type PluginHandlers struct {
adminServer interface{}
pluginMgr interface{}
}
func NewPluginHandlers(adminServer interface{}, pluginMgr interface{}) *PluginHandlers {
return &PluginHandlers{
adminServer: adminServer,
pluginMgr: pluginMgr,
}
}
// ListPluginsAPI returns list of connected plugins
func (h *PluginHandlers) ListPluginsAPI(c *gin.Context) {
result := []map[string]interface{}{}
c.JSON(http.StatusOK, result)
}
// ListJobsAPI returns jobs for a specific type
func (h *PluginHandlers) ListJobsAPI(c *gin.Context) {
jobType := c.Param("type")
result := map[string]interface{}{
"job_type": jobType,
"jobs": []interface{}{},
}
c.JSON(http.StatusOK, result)
}
// GetConfigAPI returns configuration for a job type
func (h *PluginHandlers) GetConfigAPI(c *gin.Context) {
jobType := c.Param("type")
result := map[string]interface{}{
"type": jobType,
}
c.JSON(http.StatusOK, result)
}
// SaveConfigAPI saves configuration for a job type
func (h *PluginHandlers) SaveConfigAPI(c *gin.Context) {
jobType := c.Param("type")
result := map[string]string{
"status": "saved",
"type": jobType,
}
c.JSON(http.StatusOK, result)
}
// GetDetectionHistoryAPI returns detection history for a job type
func (h *PluginHandlers) GetDetectionHistoryAPI(c *gin.Context) {
jobType := c.Param("type")
result := map[string]interface{}{
"job_type": jobType,
"records": []interface{}{},
}
c.JSON(http.StatusOK, result)
}
// GetExecutionHistoryAPI returns execution history for a job type
func (h *PluginHandlers) GetExecutionHistoryAPI(c *gin.Context) {
jobType := c.Param("type")
result := map[string]interface{}{
"job_type": jobType,
"records": []interface{}{},
}
c.JSON(http.StatusOK, result)
}
// TriggerDetectionAPI manually triggers detection
func (h *PluginHandlers) TriggerDetectionAPI(c *gin.Context) {
jobType := c.Param("type")
result := map[string]interface{}{
"status": "triggered",
"job_type": jobType,
"job_ids": []string{},
}
c.JSON(http.StatusOK, result)
}
// CancelJobAPI cancels a job
func (h *PluginHandlers) CancelJobAPI(c *gin.Context) {
jobID := c.Param("id")
result := map[string]string{
"status": "cancelled",
"job_id": jobID,
}
c.JSON(http.StatusOK, result)
}
-376
View File
@@ -1,376 +0,0 @@
package plugin
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
"time"
)
// ConfigManager handles JSON-based configuration persistence
type ConfigManager struct {
mu sync.RWMutex
configDir string
defaultConfigFile string
pluginConfigs map[string]*PluginConfig
configVersions map[string]int64
lastModified map[string]time.Time
backupDir string
maxBackups int
}
// NewConfigManager creates a new configuration manager
func NewConfigManager(configDir string) (*ConfigManager, error) {
// Ensure config directory exists
if err := os.MkdirAll(configDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create config directory: %w", err)
}
backupDir := filepath.Join(configDir, "backups")
if err := os.MkdirAll(backupDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create backup directory: %w", err)
}
return &ConfigManager{
configDir: configDir,
defaultConfigFile: filepath.Join(configDir, "plugins.json"),
pluginConfigs: make(map[string]*PluginConfig),
configVersions: make(map[string]int64),
lastModified: make(map[string]time.Time),
backupDir: backupDir,
maxBackups: 10,
}, nil
}
// SaveConfig persists a plugin configuration to disk
func (cm *ConfigManager) SaveConfig(config *PluginConfig, backup bool) error {
cm.mu.Lock()
defer cm.mu.Unlock()
if backup {
if err := cm.backupExistingConfig(config.PluginID); err != nil {
return fmt.Errorf("failed to backup config: %w", err)
}
}
configFile := filepath.Join(cm.configDir, fmt.Sprintf("%s.json", config.PluginID))
configData := map[string]interface{}{
"plugin_id": config.PluginID,
"properties": config.Properties,
"job_types": config.JobTypes,
"max_retries": config.MaxRetries,
"health_check_interval": config.HealthCheckInterval.String(),
"job_timeout": config.JobTimeout.String(),
"environment": config.Environment,
}
data, err := json.MarshalIndent(configData, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal config: %w", err)
}
if err := ioutil.WriteFile(configFile, data, 0644); err != nil {
return fmt.Errorf("failed to write config file: %w", err)
}
// Update in-memory state
cm.pluginConfigs[config.PluginID] = config
cm.configVersions[config.PluginID]++
cm.lastModified[config.PluginID] = time.Now()
return nil
}
// LoadConfig loads a plugin configuration from disk
func (cm *ConfigManager) LoadConfig(pluginID string) (*PluginConfig, error) {
cm.mu.Lock()
defer cm.mu.Unlock()
configFile := filepath.Join(cm.configDir, fmt.Sprintf("%s.json", pluginID))
data, err := ioutil.ReadFile(configFile)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
var configData map[string]interface{}
if err := json.Unmarshal(data, &configData); err != nil {
return nil, fmt.Errorf("failed to unmarshal config: %w", err)
}
config := &PluginConfig{
PluginID: pluginID,
Properties: make(map[string]string),
JobTypes: make(map[string]*JobTypeConfig),
Environment: make(map[string]string),
}
// Parse basic fields
if props, ok := configData["properties"].(map[string]interface{}); ok {
for k, v := range props {
if str, ok := v.(string); ok {
config.Properties[k] = str
}
}
}
if maxRetries, ok := configData["max_retries"].(float64); ok {
config.MaxRetries = int(maxRetries)
}
if hcInterval, ok := configData["health_check_interval"].(string); ok {
if duration, err := time.ParseDuration(hcInterval); err == nil {
config.HealthCheckInterval = duration
}
}
if timeout, ok := configData["job_timeout"].(string); ok {
if duration, err := time.ParseDuration(timeout); err == nil {
config.JobTimeout = duration
}
}
if env, ok := configData["environment"].(map[string]interface{}); ok {
for k, v := range env {
if str, ok := v.(string); ok {
config.Environment[k] = str
}
}
}
// Parse job types
if jobTypes, ok := configData["job_types"].(map[string]interface{}); ok {
for jobType, typeConfig := range jobTypes {
if typeCfg, ok := typeConfig.(map[string]interface{}); ok {
jtc := &JobTypeConfig{
Type: jobType,
Parameters: make(map[string]string),
}
if enabled, ok := typeCfg["enabled"].(bool); ok {
jtc.Enabled = enabled
}
if priority, ok := typeCfg["priority"].(float64); ok {
jtc.Priority = int(priority)
}
if interval, ok := typeCfg["interval"].(string); ok {
if duration, err := time.ParseDuration(interval); err == nil {
jtc.Interval = duration
}
}
if maxConcurrent, ok := typeCfg["max_concurrent"].(float64); ok {
jtc.MaxConcurrent = int(maxConcurrent)
}
if params, ok := typeCfg["parameters"].(map[string]interface{}); ok {
for pk, pv := range params {
if str, ok := pv.(string); ok {
jtc.Parameters[pk] = str
}
}
}
config.JobTypes[jobType] = jtc
}
}
}
cm.pluginConfigs[pluginID] = config
cm.configVersions[pluginID]++
cm.lastModified[pluginID] = time.Now()
return config, nil
}
// GetConfig retrieves a configuration from memory
func (cm *ConfigManager) GetConfig(pluginID string) (*PluginConfig, bool) {
cm.mu.RLock()
defer cm.mu.RUnlock()
config, exists := cm.pluginConfigs[pluginID]
return config, exists
}
// ListConfigs returns all loaded configurations
func (cm *ConfigManager) ListConfigs() map[string]*PluginConfig {
cm.mu.RLock()
defer cm.mu.RUnlock()
result := make(map[string]*PluginConfig)
for pluginID, config := range cm.pluginConfigs {
result[pluginID] = config
}
return result
}
// DeleteConfig removes a configuration
func (cm *ConfigManager) DeleteConfig(pluginID string) error {
cm.mu.Lock()
defer cm.mu.Unlock()
configFile := filepath.Join(cm.configDir, fmt.Sprintf("%s.json", pluginID))
if err := os.Remove(configFile); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("failed to delete config file: %w", err)
}
delete(cm.pluginConfigs, pluginID)
delete(cm.configVersions, pluginID)
delete(cm.lastModified, pluginID)
return nil
}
// GetVersion returns the version number of a configuration
func (cm *ConfigManager) GetVersion(pluginID string) int64 {
cm.mu.RLock()
defer cm.mu.RUnlock()
return cm.configVersions[pluginID]
}
// GetLastModified returns the last modification time of a configuration
func (cm *ConfigManager) GetLastModified(pluginID string) time.Time {
cm.mu.RLock()
defer cm.mu.RUnlock()
return cm.lastModified[pluginID]
}
// backupExistingConfig creates a backup of an existing configuration
func (cm *ConfigManager) backupExistingConfig(pluginID string) error {
configFile := filepath.Join(cm.configDir, fmt.Sprintf("%s.json", pluginID))
// Check if file exists
if _, err := os.Stat(configFile); os.IsNotExist(err) {
return nil // No existing config to back up
}
// Create backup filename with timestamp
backupFilename := fmt.Sprintf("%s_%d.json.bak", pluginID, time.Now().Unix())
backupFile := filepath.Join(cm.backupDir, backupFilename)
data, err := ioutil.ReadFile(configFile)
if err != nil {
return fmt.Errorf("failed to read config for backup: %w", err)
}
if err := ioutil.WriteFile(backupFile, data, 0644); err != nil {
return fmt.Errorf("failed to write backup file: %w", err)
}
// Cleanup old backups
cm.cleanupOldBackups(pluginID)
return nil
}
// cleanupOldBackups removes old backup files, keeping only maxBackups
func (cm *ConfigManager) cleanupOldBackups(pluginID string) {
pattern := filepath.Join(cm.backupDir, fmt.Sprintf("%s_*.json.bak", pluginID))
files, err := filepath.Glob(pattern)
if err != nil {
return
}
if len(files) > cm.maxBackups {
// Sort by modification time and remove oldest
for i := 0; i < len(files)-cm.maxBackups; i++ {
os.Remove(files[i])
}
}
}
// LoadAllConfigs loads all configurations from the config directory
func (cm *ConfigManager) LoadAllConfigs() error {
cm.mu.Lock()
defer cm.mu.Unlock()
files, err := filepath.Glob(filepath.Join(cm.configDir, "*.json"))
if err != nil {
return fmt.Errorf("failed to list config files: %w", err)
}
for _, file := range files {
filename := filepath.Base(file)
pluginID := filename[:len(filename)-5] // Remove .json extension
if pluginID == "plugins" {
continue // Skip main config file
}
data, err := ioutil.ReadFile(file)
if err != nil {
continue
}
var configData map[string]interface{}
if err := json.Unmarshal(data, &configData); err != nil {
continue
}
// Basic parsing (simplified)
config := &PluginConfig{
PluginID: pluginID,
Properties: make(map[string]string),
JobTypes: make(map[string]*JobTypeConfig),
Environment: make(map[string]string),
}
cm.pluginConfigs[pluginID] = config
cm.configVersions[pluginID] = 1
cm.lastModified[pluginID] = time.Now()
}
return nil
}
// ExportConfigs exports all configurations to a JSON file
func (cm *ConfigManager) ExportConfigs() (string, error) {
cm.mu.RLock()
defer cm.mu.RUnlock()
exportData := make(map[string]interface{})
for pluginID, config := range cm.pluginConfigs {
exportData[pluginID] = config
}
data, err := json.MarshalIndent(exportData, "", " ")
if err != nil {
return "", fmt.Errorf("failed to marshal configs: %w", err)
}
return string(data), nil
}
// ImportConfigs imports configurations from a JSON string
func (cm *ConfigManager) ImportConfigs(jsonData string) error {
var importData map[string]interface{}
if err := json.Unmarshal([]byte(jsonData), &importData); err != nil {
return fmt.Errorf("failed to unmarshal import data: %w", err)
}
cm.mu.Lock()
defer cm.mu.Unlock()
for pluginID, configData := range importData {
if _, ok := configData.(map[string]interface{}); ok {
config := &PluginConfig{
PluginID: pluginID,
Properties: make(map[string]string),
JobTypes: make(map[string]*JobTypeConfig),
Environment: make(map[string]string),
}
cm.pluginConfigs[pluginID] = config
cm.configVersions[pluginID]++
cm.lastModified[pluginID] = time.Now()
}
}
return nil
}
-382
View File
@@ -1,382 +0,0 @@
package plugin
import (
"fmt"
"sync"
"time"
)
// Dispatcher orchestrates job detection scheduling and dispatch
type Dispatcher struct {
mu sync.RWMutex
registry *Registry
queue *JobQueue
detectionSchedules map[string]*DetectionSchedule
jobTypeStateManagement map[string]*JobTypeState
lastDetectionTime map[string]time.Time
detectionConcurrencyLimit map[string]int
}
// DetectionSchedule holds scheduling information for a detection type
type DetectionSchedule struct {
DetectionType string
Interval time.Duration
LastExecuted time.Time
NextExecutionTime time.Time
ExecutionCount int64
FailureCount int64
AverageExecutionMs float64
}
// JobTypeState manages state for a specific job type
type JobTypeState struct {
JobType string
mu sync.RWMutex
ActiveCount int
MaxConcurrent int
PendingCount int
CompletedCount int
FailedCount int
LastError string
LastExecutionTime time.Time
AverageExecutionMs float64
ExecutionHistory []time.Duration
MaxHistorySize int
}
// NewDispatcher creates a new job dispatcher
func NewDispatcher(registry *Registry, queue *JobQueue) *Dispatcher {
return &Dispatcher{
registry: registry,
queue: queue,
detectionSchedules: make(map[string]*DetectionSchedule),
jobTypeStateManagement: make(map[string]*JobTypeState),
lastDetectionTime: make(map[string]time.Time),
detectionConcurrencyLimit: make(map[string]int),
}
}
// RegisterDetectionType registers a detection type with scheduling info
func (d *Dispatcher) RegisterDetectionType(detectionType string, interval time.Duration, maxConcurrent int) error {
d.mu.Lock()
defer d.mu.Unlock()
if _, exists := d.detectionSchedules[detectionType]; exists {
return fmt.Errorf("detection type %s already registered", detectionType)
}
d.detectionSchedules[detectionType] = &DetectionSchedule{
DetectionType: detectionType,
Interval: interval,
NextExecutionTime: time.Now(),
}
d.detectionConcurrencyLimit[detectionType] = maxConcurrent
d.jobTypeStateManagement[detectionType] = &JobTypeState{
JobType: detectionType,
MaxConcurrent: maxConcurrent,
MaxHistorySize: 100,
ExecutionHistory: make([]time.Duration, 0, 100),
}
return nil
}
// UnregisterDetectionType removes a detection type
func (d *Dispatcher) UnregisterDetectionType(detectionType string) error {
d.mu.Lock()
defer d.mu.Unlock()
if _, exists := d.detectionSchedules[detectionType]; !exists {
return fmt.Errorf("detection type %s not found", detectionType)
}
delete(d.detectionSchedules, detectionType)
delete(d.detectionConcurrencyLimit, detectionType)
delete(d.jobTypeStateManagement, detectionType)
delete(d.lastDetectionTime, detectionType)
return nil
}
// ScheduleDetections checks and schedules detection jobs that are due
func (d *Dispatcher) ScheduleDetections() []string {
d.mu.Lock()
defer d.mu.Unlock()
var scheduledJobs []string
now := time.Now()
for detectionType, schedule := range d.detectionSchedules {
if now.After(schedule.NextExecutionTime) {
// Check if we haven't exceeded concurrency limit
state := d.jobTypeStateManagement[detectionType]
state.mu.RLock()
activeCount := state.ActiveCount
maxConcurrent := state.MaxConcurrent
state.mu.RUnlock()
if activeCount >= maxConcurrent {
continue // Skip this detection type for now
}
// Create and enqueue job
jobID := fmt.Sprintf("det-%s-%d", detectionType, now.UnixNano())
job := &Job{
ID: jobID,
Type: detectionType,
State: JobStatePending,
CreatedAt: now,
}
if err := d.queue.Enqueue(job); err != nil {
continue
}
// Update schedule
schedule.NextExecutionTime = now.Add(schedule.Interval)
schedule.ExecutionCount++
d.lastDetectionTime[detectionType] = now
scheduledJobs = append(scheduledJobs, jobID)
// Update state
state.mu.Lock()
state.PendingCount++
state.mu.Unlock()
}
}
return scheduledJobs
}
// DispatchJob assigns a job to an available plugin
func (d *Dispatcher) DispatchJob(job *Job) (string, error) {
d.mu.RLock()
defer d.mu.RUnlock()
// Find plugins capable of handling this job type
plugins := d.registry.GetPluginsByCapability(job.Type)
if len(plugins) == 0 {
return "", fmt.Errorf("no plugins available for job type %s", job.Type)
}
// Find least loaded available plugin
var selectedPlugin *ConnectedPlugin
minLoad := int(^uint32(0) >> 1)
for _, plugin := range plugins {
if plugin.IsHealthy(30 * time.Second) {
plugin.mu.RLock()
if plugin.ActiveJobs < plugin.MaxConcurrentJobs && plugin.ActiveJobs < minLoad {
selectedPlugin = plugin
minLoad = plugin.ActiveJobs
}
plugin.mu.RUnlock()
}
}
if selectedPlugin == nil {
return "", fmt.Errorf("no healthy plugins available for job type %s", job.Type)
}
// Assign job to plugin
job.PluginID = selectedPlugin.ID
job.SetState(JobStateScheduled)
selectedPlugin.IncActiveJobs()
// Update job type state
state := d.jobTypeStateManagement[job.Type]
state.mu.Lock()
state.ActiveCount++
state.PendingCount--
state.mu.Unlock()
return selectedPlugin.ID, nil
}
// CompleteJob marks a job as completed
func (d *Dispatcher) CompleteJob(job *Job, result *JobResult) error {
d.mu.Lock()
defer d.mu.Unlock()
job.Result = result
job.SetState(JobStateCompleted)
// Update plugin
if plugin, err := d.registry.GetPlugin(job.PluginID); err == nil {
plugin.DecActiveJobs()
}
// Update state
if state, exists := d.jobTypeStateManagement[job.Type]; exists {
state.mu.Lock()
state.ActiveCount--
state.CompletedCount++
if job.ExecutionTime > 0 {
state.ExecutionHistory = append(state.ExecutionHistory, job.ExecutionTime)
if len(state.ExecutionHistory) > state.MaxHistorySize {
state.ExecutionHistory = state.ExecutionHistory[1:]
}
d.updateAverageExecutionTime(state)
}
state.LastExecutionTime = time.Now()
state.mu.Unlock()
}
// Update detection schedule if applicable
if schedule, exists := d.detectionSchedules[job.Type]; exists {
schedule.LastExecuted = time.Now()
}
// Record execution
record := &ExecutionRecord{
JobID: job.ID,
JobType: job.Type,
PluginID: job.PluginID,
State: job.State,
CreatedAt: job.CreatedAt,
StartedAt: job.StartedAt,
CompletedAt: job.CompletedAt,
Result: result,
}
d.queue.RecordExecution(record)
return nil
}
// FailJob marks a job as failed
func (d *Dispatcher) FailJob(job *Job, errorMsg string) error {
d.mu.Lock()
defer d.mu.Unlock()
job.LastError = errorMsg
job.SetState(JobStateFailed)
// Update plugin
if plugin, err := d.registry.GetPlugin(job.PluginID); err == nil {
plugin.DecActiveJobs()
}
// Update state
if state, exists := d.jobTypeStateManagement[job.Type]; exists {
state.mu.Lock()
state.ActiveCount--
state.FailedCount++
state.LastError = errorMsg
state.LastExecutionTime = time.Now()
state.mu.Unlock()
}
// Update detection schedule
if schedule, exists := d.detectionSchedules[job.Type]; exists {
schedule.FailureCount++
schedule.LastExecuted = time.Now()
}
// Record execution
record := &ExecutionRecord{
JobID: job.ID,
JobType: job.Type,
PluginID: job.PluginID,
State: job.State,
CreatedAt: job.CreatedAt,
StartedAt: job.StartedAt,
CompletedAt: job.CompletedAt,
LastError: errorMsg,
}
d.queue.RecordExecution(record)
return nil
}
// updateAverageExecutionTime recalculates average execution time from history
func (d *Dispatcher) updateAverageExecutionTime(state *JobTypeState) {
if len(state.ExecutionHistory) == 0 {
state.AverageExecutionMs = 0
return
}
var total int64
for _, duration := range state.ExecutionHistory {
total += duration.Milliseconds()
}
state.AverageExecutionMs = float64(total) / float64(len(state.ExecutionHistory))
}
// GetJobTypeState returns the state for a specific job type
func (d *Dispatcher) GetJobTypeState(jobType string) *JobTypeState {
d.mu.RLock()
defer d.mu.RUnlock()
if state, exists := d.jobTypeStateManagement[jobType]; exists {
return state
}
return nil
}
// GetAllJobTypeStates returns all job type states
func (d *Dispatcher) GetAllJobTypeStates() map[string]*JobTypeState {
d.mu.RLock()
defer d.mu.RUnlock()
result := make(map[string]*JobTypeState)
for jobType, state := range d.jobTypeStateManagement {
result[jobType] = state
}
return result
}
// GetDetectionSchedule returns the schedule for a detection type
func (d *Dispatcher) GetDetectionSchedule(detectionType string) *DetectionSchedule {
d.mu.RLock()
defer d.mu.RUnlock()
if schedule, exists := d.detectionSchedules[detectionType]; exists {
return schedule
}
return nil
}
// GetDueDetections returns all detection types that are due for execution
func (d *Dispatcher) GetDueDetections() []string {
d.mu.RLock()
defer d.mu.RUnlock()
var due []string
now := time.Now()
for detectionType, schedule := range d.detectionSchedules {
if now.After(schedule.NextExecutionTime) {
due = append(due, detectionType)
}
}
return due
}
// GetDispatcherStats returns overall dispatcher statistics
func (d *Dispatcher) GetDispatcherStats() map[string]interface{} {
d.mu.RLock()
defer d.mu.RUnlock()
totalActive := 0
totalCompleted := 0
totalFailed := 0
for _, state := range d.jobTypeStateManagement {
state.mu.RLock()
totalActive += state.ActiveCount
totalCompleted += state.CompletedCount
totalFailed += state.FailedCount
state.mu.RUnlock()
}
return map[string]interface{}{
"detection_types_registered": len(d.detectionSchedules),
"total_active_jobs": totalActive,
"total_completed_jobs": totalCompleted,
"total_failed_jobs": totalFailed,
"job_type_states": len(d.jobTypeStateManagement),
}
}
-457
View File
@@ -1,457 +0,0 @@
package plugin
import (
"context"
"fmt"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// GRPCServer implements the plugin service gRPC handlers
type GRPCServer struct {
mu sync.RWMutex
registry *Registry
queue *JobQueue
dispatcher *Dispatcher
configMgr *ConfigManager
streamMu sync.RWMutex
activeStreams map[string][]chan interface{}
plugin_pb.UnimplementedPluginServiceServer
plugin_pb.UnimplementedAdminQueryServiceServer
plugin_pb.UnimplementedAdminCommandServiceServer
}
// NewGRPCServer creates a new gRPC server
func NewGRPCServer(registry *Registry, queue *JobQueue, dispatcher *Dispatcher, configMgr *ConfigManager) *GRPCServer {
return &GRPCServer{
registry: registry,
queue: queue,
dispatcher: dispatcher,
configMgr: configMgr,
activeStreams: make(map[string][]chan interface{}),
}
}
// Connect registers a plugin with the master
func (gs *GRPCServer) Connect(ctx context.Context, req *plugin_pb.PluginConnectRequest) (*plugin_pb.PluginConnectResponse, error) {
if req.PluginId == "" {
return nil, fmt.Errorf("plugin_id is required")
}
// Create ConnectedPlugin instance
plugin := &ConnectedPlugin{
ID: req.PluginId,
Name: req.PluginName,
Version: req.Version,
Status: "CONNECTED",
Capabilities: req.Capabilities,
MaxConcurrentJobs: int(req.MaxConcurrentJobs),
ConnectedAt: time.Now(),
LastHeartbeat: time.Now(),
Metadata: req.Metadata,
HealthCheckInterval: 30 * time.Second,
JobTimeout: 5 * time.Minute,
}
// Register plugin
if err := gs.registry.RegisterPlugin(plugin); err != nil {
return nil, fmt.Errorf("failed to register plugin: %w", err)
}
// Load or create configuration
config, err := gs.configMgr.LoadConfig(req.PluginId)
if err != nil {
// Create default config
config = &PluginConfig{
PluginID: req.PluginId,
Properties: make(map[string]string),
JobTypes: make(map[string]*JobTypeConfig),
MaxRetries: 3,
HealthCheckInterval: 30 * time.Second,
JobTimeout: 5 * time.Minute,
Environment: make(map[string]string),
}
gs.configMgr.SaveConfig(config, false)
}
// Build response
pbConfig := &plugin_pb.PluginConfig{
PluginId: config.PluginID,
Properties: config.Properties,
MaxRetries: int32(config.MaxRetries),
Environment: config.Environment,
}
response := &plugin_pb.PluginConnectResponse{
Success: true,
Message: "Plugin registered successfully",
MasterId: "master-1",
Config: pbConfig,
AssignedTypes: req.Capabilities,
}
return response, nil
}
// ExecuteJob processes a detection or maintenance job
func (gs *GRPCServer) ExecuteJob(ctx context.Context, req *plugin_pb.ExecuteJobRequest) (*plugin_pb.ExecuteJobResponse, error) {
if req.JobId == "" || req.JobType == "" {
return nil, fmt.Errorf("job_id and job_type are required")
}
response := &plugin_pb.ExecuteJobResponse{
JobId: req.JobId,
Status: plugin_pb.ExecutionStatus_EXECUTION_STATUS_ACCEPTED,
Message: "Job accepted for execution",
}
return response, nil
}
// ReportHealth processes health reports from plugins
func (gs *GRPCServer) ReportHealth(ctx context.Context, report *plugin_pb.HealthReport) (*plugin_pb.HealthReportResponse, error) {
if report.PluginId == "" {
return nil, fmt.Errorf("plugin_id is required")
}
// Update heartbeat
if err := gs.registry.UpdateHeartbeat(report.PluginId); err != nil {
return nil, fmt.Errorf("plugin not found: %w", err)
}
// Update plugin stats
if plugin, err := gs.registry.GetPlugin(report.PluginId); err == nil {
plugin.mu.Lock()
plugin.ActiveJobs = int(report.ActiveJobs)
plugin.CPUUsagePercent = float64(report.CpuPercent)
plugin.MemoryUsageBytes = report.MemoryBytes
plugin.mu.Unlock()
}
return &plugin_pb.HealthReportResponse{
Acknowledged: true,
Feedback: "Health report received",
}, nil
}
// GetConfig retrieves the latest configuration
func (gs *GRPCServer) GetConfig(ctx context.Context, req *plugin_pb.GetConfigRequest) (*plugin_pb.GetConfigResponse, error) {
if req.PluginId == "" {
return nil, fmt.Errorf("plugin_id is required")
}
config, exists := gs.configMgr.GetConfig(req.PluginId)
if !exists {
return nil, fmt.Errorf("config not found for plugin: %s", req.PluginId)
}
pbConfig := &plugin_pb.PluginConfig{
PluginId: config.PluginID,
Properties: config.Properties,
MaxRetries: int32(config.MaxRetries),
Environment: config.Environment,
}
response := &plugin_pb.GetConfigResponse{
Config: pbConfig,
Version: gs.configMgr.GetVersion(req.PluginId),
}
return response, nil
}
// SubmitResult sends job execution results back to master
func (gs *GRPCServer) SubmitResult(ctx context.Context, req *plugin_pb.JobResultRequest) (*plugin_pb.JobResultResponse, error) {
if req.JobId == "" {
return nil, fmt.Errorf("job_id is required")
}
actions := []string{}
// Process results based on job status
switch req.Status {
case plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED:
actions = append(actions, "ARCHIVED")
case plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED:
actions = append(actions, "RETRY", "NOTIFY_ADMIN")
}
response := &plugin_pb.JobResultResponse{
Acknowledged: true,
ActionsToTake: actions,
}
return response, nil
}
// GetPluginStats returns statistics for all connected plugins
func (gs *GRPCServer) GetPluginStats(ctx context.Context, req *plugin_pb.GetPluginStatsRequest) (*plugin_pb.GetPluginStatsResponse, error) {
response := &plugin_pb.GetPluginStatsResponse{
Stats: []*plugin_pb.PluginStats{},
}
var plugins []*ConnectedPlugin
if req.PluginId != "" {
if plugin, err := gs.registry.GetPlugin(req.PluginId); err == nil {
plugins = append(plugins, plugin)
}
} else {
plugins = gs.registry.ListPlugins(false)
}
for _, plugin := range plugins {
stat := &plugin_pb.PluginStats{
PluginId: plugin.ID,
Status: plugin.Status,
ActiveJobs: int32(plugin.ActiveJobs),
CompletedJobs: int32(plugin.CompletedJobs),
FailedJobs: int32(plugin.FailedJobs),
TotalDetections: plugin.TotalDetections,
AvgExecutionTimeMs: float32(plugin.AvgExecutionTimeMs),
CpuUsagePercent: float32(plugin.CPUUsagePercent),
MemoryUsageBytes: plugin.MemoryUsageBytes,
UptimeSeconds: int32(time.Since(plugin.ConnectedAt).Seconds()),
}
response.Stats = append(response.Stats, stat)
}
return response, nil
}
// ListPlugins returns information about all registered plugins
func (gs *GRPCServer) ListPlugins(ctx context.Context, req *plugin_pb.ListPluginsRequest) (*plugin_pb.ListPluginsResponse, error) {
response := &plugin_pb.ListPluginsResponse{
Plugins: []*plugin_pb.PluginInfo{},
}
plugins := gs.registry.ListPlugins(!req.IncludeDisabled)
for _, plugin := range plugins {
// Filter by capability if specified
if len(req.FilterByCapability) > 0 {
hasCapability := false
for _, filterCap := range req.FilterByCapability {
for _, cap := range plugin.Capabilities {
if cap == filterCap {
hasCapability = true
break
}
}
if hasCapability {
break
}
}
if !hasCapability {
continue
}
}
info := &plugin_pb.PluginInfo{
PluginId: plugin.ID,
Name: plugin.Name,
Version: plugin.Version,
Status: plugin.Status,
Capabilities: plugin.Capabilities,
MaxConcurrentJobs: int32(plugin.MaxConcurrentJobs),
ActiveJobs: int32(plugin.ActiveJobs),
Metadata: plugin.Metadata,
}
response.Plugins = append(response.Plugins, info)
}
return response, nil
}
// ListJobs returns current and historical job information
func (gs *GRPCServer) ListJobs(ctx context.Context, req *plugin_pb.ListJobsRequest) (*plugin_pb.ListJobsResponse, error) {
response := &plugin_pb.ListJobsResponse{
Jobs: []*plugin_pb.JobInfo{},
}
var records []*ExecutionRecord
limit := int(req.Limit)
if limit <= 0 {
limit = 100
}
if req.PluginId != "" {
records = gs.queue.GetHistoryForPlugin(req.PluginId, limit)
} else {
records = gs.queue.GetHistory(limit)
}
for _, record := range records {
info := &plugin_pb.JobInfo{
JobId: record.JobID,
JobType: record.JobType,
PluginId: record.PluginID,
RetryCount: int32(record.RetryCount),
LastError: record.LastError,
}
response.Jobs = append(response.Jobs, info)
}
response.TotalCount = int32(gs.queue.HistorySize())
return response, nil
}
// GetJobStatus returns detailed status of a specific job
func (gs *GRPCServer) GetJobStatus(ctx context.Context, req *plugin_pb.GetJobStatusRequest) (*plugin_pb.GetJobStatusResponse, error) {
if req.JobId == "" {
return nil, fmt.Errorf("job_id is required")
}
// Search in queue history
records := gs.queue.GetHistory(10000)
for _, record := range records {
if record.JobID == req.JobId {
response := &plugin_pb.GetJobStatusResponse{
DetailedStatus: record.State.String(),
}
return response, nil
}
}
return nil, fmt.Errorf("job not found: %s", req.JobId)
}
// GetPluginLogs returns logs from a specific plugin (stub implementation)
func (gs *GRPCServer) GetPluginLogs(ctx context.Context, req *plugin_pb.GetPluginLogsRequest) (*plugin_pb.GetPluginLogsResponse, error) {
response := &plugin_pb.GetPluginLogsResponse{
Entries: []*plugin_pb.LogEntry{},
}
return response, nil
}
// SaveConfig persists plugin configuration
func (gs *GRPCServer) SaveConfig(ctx context.Context, req *plugin_pb.SaveConfigRequest) (*plugin_pb.SaveConfigResponse, error) {
if req.Config == nil {
return nil, fmt.Errorf("config is required")
}
// Convert from protobuf config to internal config
config := &PluginConfig{
PluginID: req.Config.PluginId,
Properties: req.Config.Properties,
MaxRetries: int(req.Config.MaxRetries),
Environment: req.Config.Environment,
JobTypes: make(map[string]*JobTypeConfig),
}
if err := gs.configMgr.SaveConfig(config, req.BackupExisting); err != nil {
return nil, fmt.Errorf("failed to save config: %w", err)
}
response := &plugin_pb.SaveConfigResponse{
Success: true,
Message: "Configuration saved successfully",
ConfigVersion: gs.configMgr.GetVersion(req.Config.PluginId),
}
return response, nil
}
// ReloadConfig reloads configuration without restarting
func (gs *GRPCServer) ReloadConfig(ctx context.Context, req *plugin_pb.ReloadConfigRequest) (*plugin_pb.ReloadConfigResponse, error) {
if req.PluginId == "" {
return nil, fmt.Errorf("plugin_id is required")
}
if _, err := gs.configMgr.LoadConfig(req.PluginId); err != nil {
return nil, fmt.Errorf("failed to reload config: %w", err)
}
response := &plugin_pb.ReloadConfigResponse{
Success: true,
Message: "Configuration reloaded successfully",
}
return response, nil
}
// EnablePlugin enables a specific plugin
func (gs *GRPCServer) EnablePlugin(ctx context.Context, req *plugin_pb.EnablePluginRequest) (*plugin_pb.EnablePluginResponse, error) {
if err := gs.registry.UpdatePluginStatus(req.PluginId, "ENABLED"); err != nil {
return nil, fmt.Errorf("failed to enable plugin: %w", err)
}
response := &plugin_pb.EnablePluginResponse{
Success: true,
Message: "Plugin enabled successfully",
}
return response, nil
}
// DisablePlugin disables a specific plugin
func (gs *GRPCServer) DisablePlugin(ctx context.Context, req *plugin_pb.DisablePluginRequest) (*plugin_pb.DisablePluginResponse, error) {
if err := gs.registry.UpdatePluginStatus(req.PluginId, "DISABLED"); err != nil {
return nil, fmt.Errorf("failed to disable plugin: %w", err)
}
response := &plugin_pb.DisablePluginResponse{
Success: true,
Message: "Plugin disabled successfully",
}
return response, nil
}
// TriggerDetection manually triggers a detection for specific types
func (gs *GRPCServer) TriggerDetection(ctx context.Context, req *plugin_pb.TriggerDetectionRequest) (*plugin_pb.TriggerDetectionResponse, error) {
response := &plugin_pb.TriggerDetectionResponse{
Success: true,
TriggeredJobIds: []string{},
}
for _, detectionType := range req.DetectionTypes {
jobID := fmt.Sprintf("trig-%s-%d", detectionType, time.Now().UnixNano())
job := &Job{
ID: jobID,
Type: detectionType,
State: JobStatePending,
CreatedAt: time.Now(),
}
if err := gs.queue.Enqueue(job); err != nil {
continue
}
response.TriggeredJobIds = append(response.TriggeredJobIds, jobID)
}
return response, nil
}
// CancelJob cancels a running job
func (gs *GRPCServer) CancelJob(ctx context.Context, req *plugin_pb.CancelJobRequest) (*plugin_pb.CancelJobResponse, error) {
if req.JobId == "" {
return nil, fmt.Errorf("job_id is required")
}
if gs.queue.RemoveJob(req.JobId) {
return &plugin_pb.CancelJobResponse{
Success: true,
Message: "Job cancelled successfully",
}, nil
}
return &plugin_pb.CancelJobResponse{
Success: false,
Message: "Job not found or already completed",
}, nil
}
// PurgeHistory clears job history
func (gs *GRPCServer) PurgeHistory(ctx context.Context, req *plugin_pb.PurgeHistoryRequest) (*plugin_pb.PurgeHistoryResponse, error) {
beforeTime := time.Unix(0, req.BeforeTimestampMs*1000000)
deleted := gs.queue.PurgeOldHistory(beforeTime)
response := &plugin_pb.PurgeHistoryResponse{
Success: true,
RecordsDeleted: int32(deleted),
}
return response, nil
}
-298
View File
@@ -1,298 +0,0 @@
package plugin
import (
"container/heap"
"fmt"
"sync"
"time"
)
// JobQueue manages job queueing with priority, deduplication, retry and history
type JobQueue struct {
mu sync.RWMutex
priorityQueue *PriorityQueue
seenJobs map[string]bool // For deduplication
jobHistory []*ExecutionRecord
maxHistorySize int
deduplicationTTL time.Duration
lastSeenJob map[string]time.Time
}
// PriorityQueue implements heap.Interface for job ordering
type PriorityQueue []*Job
func (pq PriorityQueue) Len() int { return len(pq) }
func (pq PriorityQueue) Less(i, j int) bool {
// Higher priority jobs come first
if pq[i].Type != pq[j].Type {
return pq[i].Type < pq[j].Type
}
// If same type, earlier creation time comes first
return pq[i].CreatedAt.Before(pq[j].CreatedAt)
}
func (pq PriorityQueue) Swap(i, j int) {
pq[i], pq[j] = pq[j], pq[i]
}
func (pq *PriorityQueue) Push(x interface{}) {
*pq = append(*pq, x.(*Job))
}
func (pq *PriorityQueue) Pop() interface{} {
old := *pq
n := len(old)
item := old[n-1]
*pq = old[0 : n-1]
return item
}
// NewJobQueue creates a new job queue
func NewJobQueue(maxHistorySize int, deduplicationTTL time.Duration) *JobQueue {
jq := &JobQueue{
priorityQueue: &PriorityQueue{},
seenJobs: make(map[string]bool),
jobHistory: make([]*ExecutionRecord, 0, maxHistorySize),
maxHistorySize: maxHistorySize,
deduplicationTTL: deduplicationTTL,
lastSeenJob: make(map[string]time.Time),
}
heap.Init(jq.priorityQueue)
return jq
}
// Enqueue adds a job to the queue with deduplication
func (jq *JobQueue) Enqueue(job *Job) error {
jq.mu.Lock()
defer jq.mu.Unlock()
// Check for duplicates within TTL window
if lastSeen, exists := jq.lastSeenJob[job.ID]; exists {
if time.Since(lastSeen) < jq.deduplicationTTL {
return fmt.Errorf("job %s already enqueued recently", job.ID)
}
}
job.SetState(JobStatePending)
heap.Push(jq.priorityQueue, job)
jq.seenJobs[job.ID] = true
jq.lastSeenJob[job.ID] = time.Now()
return nil
}
// Dequeue retrieves the next job from the queue
func (jq *JobQueue) Dequeue() *Job {
jq.mu.Lock()
defer jq.mu.Unlock()
if jq.priorityQueue.Len() == 0 {
return nil
}
job := heap.Pop(jq.priorityQueue).(*Job)
return job
}
// Peek returns the next job without removing it
func (jq *JobQueue) Peek() *Job {
jq.mu.RLock()
defer jq.mu.RUnlock()
if jq.priorityQueue.Len() == 0 {
return nil
}
return (*jq.priorityQueue)[0]
}
// Size returns the current queue size
func (jq *JobQueue) Size() int {
jq.mu.RLock()
defer jq.mu.RUnlock()
return jq.priorityQueue.Len()
}
// RecordExecution adds an execution record to history
func (jq *JobQueue) RecordExecution(record *ExecutionRecord) {
jq.mu.Lock()
defer jq.mu.Unlock()
jq.jobHistory = append(jq.jobHistory, record)
// Keep history size bounded
if len(jq.jobHistory) > jq.maxHistorySize {
// Remove oldest entries
removeCount := len(jq.jobHistory) - jq.maxHistorySize
jq.jobHistory = jq.jobHistory[removeCount:]
}
}
// GetHistory returns job execution history
func (jq *JobQueue) GetHistory(limit int) []*ExecutionRecord {
jq.mu.RLock()
defer jq.mu.RUnlock()
if limit <= 0 || limit > len(jq.jobHistory) {
limit = len(jq.jobHistory)
}
// Return the most recent entries
startIdx := len(jq.jobHistory) - limit
if startIdx < 0 {
startIdx = 0
}
result := make([]*ExecutionRecord, limit)
copy(result, jq.jobHistory[startIdx:])
return result
}
// GetHistoryForPlugin returns history for a specific plugin
func (jq *JobQueue) GetHistoryForPlugin(pluginID string, limit int) []*ExecutionRecord {
jq.mu.RLock()
defer jq.mu.RUnlock()
var result []*ExecutionRecord
for i := len(jq.jobHistory) - 1; i >= 0 && len(result) < limit; i-- {
if jq.jobHistory[i].PluginID == pluginID {
result = append(result, jq.jobHistory[i])
}
}
return result
}
// GetHistoryForJobType returns history for a specific job type
func (jq *JobQueue) GetHistoryForJobType(jobType string, limit int) []*ExecutionRecord {
jq.mu.RLock()
defer jq.mu.RUnlock()
var result []*ExecutionRecord
for i := len(jq.jobHistory) - 1; i >= 0 && len(result) < limit; i-- {
if jq.jobHistory[i].JobType == jobType {
result = append(result, jq.jobHistory[i])
}
}
return result
}
// ClearHistory removes all execution history
func (jq *JobQueue) ClearHistory() {
jq.mu.Lock()
defer jq.mu.Unlock()
jq.jobHistory = make([]*ExecutionRecord, 0, jq.maxHistorySize)
}
// PurgeOldHistory removes history entries older than the specified time
func (jq *JobQueue) PurgeOldHistory(beforeTime time.Time) int {
jq.mu.Lock()
defer jq.mu.Unlock()
removed := 0
newHistory := make([]*ExecutionRecord, 0)
for _, record := range jq.jobHistory {
if record.CreatedAt.After(beforeTime) {
newHistory = append(newHistory, record)
} else {
removed++
}
}
jq.jobHistory = newHistory
return removed
}
// HistorySize returns the number of records in history
func (jq *JobQueue) HistorySize() int {
jq.mu.RLock()
defer jq.mu.RUnlock()
return len(jq.jobHistory)
}
// RetryJob re-enqueues a failed job up to maxRetries times
func (jq *JobQueue) RetryJob(job *Job, maxRetries int) error {
jq.mu.Lock()
defer jq.mu.Unlock()
if job.RetryCount >= maxRetries {
return fmt.Errorf("job %s exceeded max retries (%d)", job.ID, maxRetries)
}
job.RetryCount++
job.SetState(JobStatePending)
heap.Push(jq.priorityQueue, job)
return nil
}
// GetExecutionStats returns statistics about job executions
func (jq *JobQueue) GetExecutionStats() map[string]interface{} {
jq.mu.RLock()
defer jq.mu.RUnlock()
completed := 0
failed := 0
totalExecutionTime := int64(0)
for _, record := range jq.jobHistory {
switch record.State {
case JobStateCompleted:
completed++
case JobStateFailed:
failed++
}
if record.CompletedAt != nil && record.StartedAt != nil {
totalExecutionTime += record.CompletedAt.Sub(*record.StartedAt).Milliseconds()
}
}
avgExecutionTime := int64(0)
if completed+failed > 0 {
avgExecutionTime = totalExecutionTime / int64(completed+failed)
}
return map[string]interface{}{
"total_history": len(jq.jobHistory),
"completed_jobs": completed,
"failed_jobs": failed,
"avg_execution_time_ms": avgExecutionTime,
"current_queue_size": jq.priorityQueue.Len(),
}
}
// GetQueuedJobs returns all jobs currently in the queue
func (jq *JobQueue) GetQueuedJobs() []*Job {
jq.mu.RLock()
defer jq.mu.RUnlock()
result := make([]*Job, len(*jq.priorityQueue))
copy(result, *jq.priorityQueue)
return result
}
// RemoveJob removes a specific job from the queue
func (jq *JobQueue) RemoveJob(jobID string) bool {
jq.mu.Lock()
defer jq.mu.Unlock()
for i, job := range *jq.priorityQueue {
if job.ID == jobID {
heap.Remove(jq.priorityQueue, i)
return true
}
}
return false
}
// PurgeQueuedJobs clears all pending jobs from the queue
func (jq *JobQueue) PurgeQueuedJobs() int {
jq.mu.Lock()
defer jq.mu.Unlock()
count := jq.priorityQueue.Len()
*jq.priorityQueue = PriorityQueue{}
heap.Init(jq.priorityQueue)
return count
}
-498
View File
@@ -1,498 +0,0 @@
package plugin
import (
"context"
"fmt"
"sync"
"time"
)
// Manager is the main component orchestrating the plugin system
type Manager struct {
mu sync.RWMutex
registry *Registry
queue *JobQueue
dispatcher *Dispatcher
configMgr *ConfigManager
grpcServer *GRPCServer
isRunning bool
stopChan chan bool
scheduleTicker *time.Ticker
healthCheckTicker *time.Ticker
detectionTicker *time.Ticker
wg sync.WaitGroup
ctx context.Context
cancel context.CancelFunc
config *ManagerConfig
}
// ManagerConfig holds configuration for the plugin manager
type ManagerConfig struct {
ConfigDir string
ScheduleInterval time.Duration
HealthCheckInterval time.Duration
DetectionInterval time.Duration
MaxQueueSize int
MaxHistorySize int
DeduplicationTTL time.Duration
HealthCheckTimeout time.Duration
FailureDetectionWindow time.Duration
FailureThreshold int
}
// DefaultManagerConfig returns default configuration
func DefaultManagerConfig(configDir string) *ManagerConfig {
return &ManagerConfig{
ConfigDir: configDir,
ScheduleInterval: 5 * time.Second,
HealthCheckInterval: 30 * time.Second,
DetectionInterval: 10 * time.Second,
MaxQueueSize: 10000,
MaxHistorySize: 5000,
DeduplicationTTL: 1 * time.Minute,
HealthCheckTimeout: 90 * time.Second,
FailureDetectionWindow: 5 * time.Minute,
FailureThreshold: 3,
}
}
// NewManager creates a new plugin manager instance
func NewManager(config *ManagerConfig) (*Manager, error) {
if config == nil {
return nil, fmt.Errorf("config is required")
}
// Create configuration manager
configMgr, err := NewConfigManager(config.ConfigDir)
if err != nil {
return nil, fmt.Errorf("failed to create config manager: %w", err)
}
// Create registry
registry := NewRegistry(
config.HealthCheckTimeout,
config.FailureDetectionWindow,
config.FailureThreshold,
)
// Create job queue
queue := NewJobQueue(config.MaxHistorySize, config.DeduplicationTTL)
// Create dispatcher
dispatcher := NewDispatcher(registry, queue)
// Create gRPC server
grpcServer := NewGRPCServer(registry, queue, dispatcher, configMgr)
ctx, cancel := context.WithCancel(context.Background())
manager := &Manager{
registry: registry,
queue: queue,
dispatcher: dispatcher,
configMgr: configMgr,
grpcServer: grpcServer,
stopChan: make(chan bool),
ctx: ctx,
cancel: cancel,
config: config,
}
return manager, nil
}
// Start initializes and starts the plugin manager
func (m *Manager) Start() error {
m.mu.Lock()
if m.isRunning {
m.mu.Unlock()
return fmt.Errorf("manager already running")
}
m.isRunning = true
m.mu.Unlock()
// Load existing configurations
if err := m.configMgr.LoadAllConfigs(); err != nil {
m.isRunning = false
return fmt.Errorf("failed to load configurations: %w", err)
}
// Start background tasks
m.wg.Add(3)
go m.schedulerLoop()
go m.healthCheckLoop()
go m.detectionLoop()
return nil
}
// Stop gracefully stops the plugin manager
func (m *Manager) Stop() error {
m.mu.Lock()
if !m.isRunning {
m.mu.Unlock()
return fmt.Errorf("manager not running")
}
m.isRunning = false
m.mu.Unlock()
// Signal all goroutines to stop
m.cancel()
close(m.stopChan)
// Wait for all goroutines to finish
done := make(chan struct{})
go func() {
m.wg.Wait()
close(done)
}()
select {
case <-done:
return nil
case <-time.After(30 * time.Second):
return fmt.Errorf("timeout waiting for manager to stop")
}
}
// schedulerLoop periodically schedules detection jobs
func (m *Manager) schedulerLoop() {
defer m.wg.Done()
m.scheduleTicker = time.NewTicker(m.config.ScheduleInterval)
defer m.scheduleTicker.Stop()
for {
select {
case <-m.ctx.Done():
return
case <-m.stopChan:
return
case <-m.scheduleTicker.C:
m.performScheduling()
}
}
}
// performScheduling executes the scheduling logic
func (m *Manager) performScheduling() {
scheduledJobs := m.dispatcher.ScheduleDetections()
if len(scheduledJobs) > 0 {
// Jobs have been queued for processing
}
}
// healthCheckLoop periodically checks plugin health
func (m *Manager) healthCheckLoop() {
defer m.wg.Done()
m.healthCheckTicker = time.NewTicker(m.config.HealthCheckInterval)
defer m.healthCheckTicker.Stop()
for {
select {
case <-m.ctx.Done():
return
case <-m.stopChan:
return
case <-m.healthCheckTicker.C:
m.performHealthCheck()
}
}
}
// performHealthCheck checks health of all plugins
func (m *Manager) performHealthCheck() {
plugins := m.registry.ListPlugins(true)
for _, plugin := range plugins {
isHealthy, _ := m.registry.HealthCheck(plugin.ID)
if !isHealthy {
// Check if exceeded threshold
if m.registry.HasExceededFailureThreshold(plugin.ID) {
m.registry.UnregisterPlugin(plugin.ID)
}
}
}
}
// detectionLoop periodically triggers detection execution
func (m *Manager) detectionLoop() {
defer m.wg.Done()
m.detectionTicker = time.NewTicker(m.config.DetectionInterval)
defer m.detectionTicker.Stop()
for {
select {
case <-m.ctx.Done():
return
case <-m.stopChan:
return
case <-m.detectionTicker.C:
m.processDetectionJobs()
}
}
}
// processDetectionJobs dequeues and dispatches pending jobs
func (m *Manager) processDetectionJobs() {
for {
job := m.queue.Dequeue()
if job == nil {
break
}
// Dispatch job to available plugin
pluginID, err := m.dispatcher.DispatchJob(job)
if err != nil {
// Requeue job if dispatch failed
m.queue.Enqueue(job)
break
}
job.PluginID = pluginID
}
}
// IsRunning returns whether the manager is currently running
func (m *Manager) IsRunning() bool {
m.mu.RLock()
defer m.mu.RUnlock()
return m.isRunning
}
// RegisterDetectionType registers a new detection type
func (m *Manager) RegisterDetectionType(detectionType string, interval time.Duration, maxConcurrent int) error {
return m.dispatcher.RegisterDetectionType(detectionType, interval, maxConcurrent)
}
// UnregisterDetectionType unregisters a detection type
func (m *Manager) UnregisterDetectionType(detectionType string) error {
return m.dispatcher.UnregisterDetectionType(detectionType)
}
// GetStats returns overall statistics
func (m *Manager) GetStats() map[string]interface{} {
return map[string]interface{}{
"registry": m.registry.GetStats(),
"queue": m.queue.GetExecutionStats(),
"dispatcher": m.dispatcher.GetDispatcherStats(),
"running": m.IsRunning(),
}
}
// GetPluginStats returns statistics for a specific plugin
func (m *Manager) GetPluginStats(pluginID string) (map[string]interface{}, error) {
plugin, err := m.registry.GetPlugin(pluginID)
if err != nil {
return nil, err
}
plugin.mu.RLock()
defer plugin.mu.RUnlock()
return map[string]interface{}{
"id": plugin.ID,
"name": plugin.Name,
"version": plugin.Version,
"status": plugin.Status,
"capabilities": plugin.Capabilities,
"active_jobs": plugin.ActiveJobs,
"completed_jobs": plugin.CompletedJobs,
"failed_jobs": plugin.FailedJobs,
"total_detections": plugin.TotalDetections,
"avg_execution_time_ms": plugin.AvgExecutionTimeMs,
"cpu_usage_percent": plugin.CPUUsagePercent,
"memory_usage_bytes": plugin.MemoryUsageBytes,
"connected_at": plugin.ConnectedAt,
"last_heartbeat": plugin.LastHeartbeat,
"uptime_seconds": int(time.Since(plugin.ConnectedAt).Seconds()),
}, nil
}
// ListPlugins returns all registered plugins
func (m *Manager) ListPlugins(includeUnhealthy bool) []*ConnectedPlugin {
return m.registry.ListPlugins(includeUnhealthy)
}
// ListJobs returns job history
func (m *Manager) ListJobs(limit int) []*ExecutionRecord {
return m.queue.GetHistory(limit)
}
// ListJobsForPlugin returns jobs for a specific plugin
func (m *Manager) ListJobsForPlugin(pluginID string, limit int) []*ExecutionRecord {
return m.queue.GetHistoryForPlugin(pluginID, limit)
}
// ListJobsForType returns jobs for a specific type
func (m *Manager) ListJobsForType(jobType string, limit int) []*ExecutionRecord {
return m.queue.GetHistoryForJobType(jobType, limit)
}
// TriggerDetection manually triggers detection for specific types
func (m *Manager) TriggerDetection(detectionTypes []string) ([]string, error) {
var jobIDs []string
for _, detectionType := range detectionTypes {
jobID := fmt.Sprintf("manual-%s-%d", detectionType, time.Now().UnixNano())
job := &Job{
ID: jobID,
Type: detectionType,
State: JobStatePending,
CreatedAt: time.Now(),
}
if err := m.queue.Enqueue(job); err != nil {
continue
}
jobIDs = append(jobIDs, jobID)
}
return jobIDs, nil
}
// GetJobStatus returns the status of a specific job
func (m *Manager) GetJobStatus(jobID string) (*ExecutionRecord, error) {
records := m.queue.GetHistory(10000)
for _, record := range records {
if record.JobID == jobID {
return record, nil
}
}
return nil, fmt.Errorf("job not found: %s", jobID)
}
// CancelJob cancels a pending or scheduled job
func (m *Manager) CancelJob(jobID string) error {
if !m.queue.RemoveJob(jobID) {
return fmt.Errorf("job not found or already completed: %s", jobID)
}
return nil
}
// PurgeHistory removes old job history
func (m *Manager) PurgeHistory(beforeTime time.Time) int {
return m.queue.PurgeOldHistory(beforeTime)
}
// SaveConfig saves plugin configuration
func (m *Manager) SaveConfig(config *PluginConfig, backup bool) error {
return m.configMgr.SaveConfig(config, backup)
}
// LoadConfig loads plugin configuration
func (m *Manager) LoadConfig(pluginID string) (*PluginConfig, error) {
config, err := m.configMgr.LoadConfig(pluginID)
if err != nil {
return nil, fmt.Errorf("failed to load config: %w", err)
}
return config, nil
}
// ListConfigs returns all loaded configurations
func (m *Manager) ListConfigs() map[string]*PluginConfig {
return m.configMgr.ListConfigs()
}
// DeleteConfig deletes a configuration
func (m *Manager) DeleteConfig(pluginID string) error {
return m.configMgr.DeleteConfig(pluginID)
}
// GetRegistry returns the plugin registry
func (m *Manager) GetRegistry() *Registry {
return m.registry
}
// GetQueue returns the job queue
func (m *Manager) GetQueue() *JobQueue {
return m.queue
}
// GetDispatcher returns the dispatcher
func (m *Manager) GetDispatcher() *Dispatcher {
return m.dispatcher
}
// GetGRPCServer returns the gRPC server
func (m *Manager) GetGRPCServer() *GRPCServer {
return m.grpcServer
}
// GetDetectionHistory returns detection history for a job type
func (m *Manager) GetDetectionHistory(jobType string) []DetectionRecord {
m.mu.RLock()
defer m.mu.RUnlock()
configs := m.configMgr.ListConfigs()
for _, cfg := range configs {
if jobCfg, ok := cfg.GetJobTypeConfig(jobType); ok {
cfg.mu.RLock()
defer cfg.mu.RUnlock()
history := make([]DetectionRecord, len(jobCfg.DetectionHistory))
copy(history, jobCfg.DetectionHistory)
return history
}
}
return []DetectionRecord{}
}
// GetExecutionHistory returns execution history for a job type
func (m *Manager) GetExecutionHistory(jobType string) []ExecutionRecord {
m.mu.RLock()
defer m.mu.RUnlock()
configs := m.configMgr.ListConfigs()
for _, cfg := range configs {
if jobCfg, ok := cfg.GetJobTypeConfig(jobType); ok {
cfg.mu.RLock()
defer cfg.mu.RUnlock()
history := make([]ExecutionRecord, len(jobCfg.ExecutionHistory))
copy(history, jobCfg.ExecutionHistory)
return history
}
}
return []ExecutionRecord{}
}
// RecordDetection adds a detection record to history
func (m *Manager) RecordDetection(jobType string, record *DetectionRecord) {
m.mu.RLock()
defer m.mu.RUnlock()
configs := m.configMgr.ListConfigs()
for _, cfg := range configs {
if jobCfg, ok := cfg.GetJobTypeConfig(jobType); ok {
cfg.mu.Lock()
maxSize := 50
jobCfg.DetectionHistory = append([]DetectionRecord{*record}, jobCfg.DetectionHistory...)
if len(jobCfg.DetectionHistory) > maxSize {
jobCfg.DetectionHistory = jobCfg.DetectionHistory[:maxSize]
}
cfg.mu.Unlock()
break
}
}
}
// RecordExecution adds an execution record to history
func (m *Manager) RecordExecution(jobType string, record *ExecutionRecord) {
m.mu.RLock()
defer m.mu.RUnlock()
configs := m.configMgr.ListConfigs()
for _, cfg := range configs {
if jobCfg, ok := cfg.GetJobTypeConfig(jobType); ok {
cfg.mu.Lock()
maxSize := 100
jobCfg.ExecutionHistory = append([]ExecutionRecord{*record}, jobCfg.ExecutionHistory...)
if len(jobCfg.ExecutionHistory) > maxSize {
jobCfg.ExecutionHistory = jobCfg.ExecutionHistory[:maxSize]
}
cfg.mu.Unlock()
break
}
}
}
-311
View File
@@ -1,311 +0,0 @@
package plugin
import (
"fmt"
"sync"
"time"
)
// Registry manages plugin registration and lifecycle
type Registry struct {
mu sync.RWMutex
plugins map[string]*ConnectedPlugin
capabilityIndex map[string][]string // Maps capability to plugin IDs
healthCheckTimeout time.Duration
failureDetectionWindow time.Duration
failureThreshold int
pluginFailureCount map[string]int
}
// NewRegistry creates a new plugin registry
func NewRegistry(healthCheckTimeout, failureDetectionWindow time.Duration, failureThreshold int) *Registry {
return &Registry{
plugins: make(map[string]*ConnectedPlugin),
capabilityIndex: make(map[string][]string),
healthCheckTimeout: healthCheckTimeout,
failureDetectionWindow: failureDetectionWindow,
failureThreshold: failureThreshold,
pluginFailureCount: make(map[string]int),
}
}
// RegisterPlugin adds a plugin to the registry
func (r *Registry) RegisterPlugin(plugin *ConnectedPlugin) error {
r.mu.Lock()
defer r.mu.Unlock()
if _, exists := r.plugins[plugin.ID]; exists {
return fmt.Errorf("plugin %s already registered", plugin.ID)
}
r.plugins[plugin.ID] = plugin
r.pluginFailureCount[plugin.ID] = 0
// Build capability index
for _, cap := range plugin.Capabilities {
r.capabilityIndex[cap] = append(r.capabilityIndex[cap], plugin.ID)
}
return nil
}
// UnregisterPlugin removes a plugin from the registry
func (r *Registry) UnregisterPlugin(pluginID string) error {
r.mu.Lock()
defer r.mu.Unlock()
plugin, exists := r.plugins[pluginID]
if !exists {
return fmt.Errorf("plugin %s not found", pluginID)
}
// Remove from capability index
for _, cap := range plugin.Capabilities {
for i, id := range r.capabilityIndex[cap] {
if id == pluginID {
r.capabilityIndex[cap] = append(r.capabilityIndex[cap][:i], r.capabilityIndex[cap][i+1:]...)
break
}
}
}
delete(r.plugins, pluginID)
delete(r.pluginFailureCount, pluginID)
return nil
}
// GetPlugin retrieves a plugin by ID
func (r *Registry) GetPlugin(pluginID string) (*ConnectedPlugin, error) {
r.mu.RLock()
defer r.mu.RUnlock()
plugin, exists := r.plugins[pluginID]
if !exists {
return nil, fmt.Errorf("plugin %s not found", pluginID)
}
return plugin, nil
}
// GetPluginsByCapability returns all plugins with a specific capability
func (r *Registry) GetPluginsByCapability(capability string) []*ConnectedPlugin {
r.mu.RLock()
defer r.mu.RUnlock()
pluginIDs, exists := r.capabilityIndex[capability]
if !exists {
return []*ConnectedPlugin{}
}
var result []*ConnectedPlugin
for _, id := range pluginIDs {
if plugin, ok := r.plugins[id]; ok {
result = append(result, plugin)
}
}
return result
}
// ListPlugins returns all registered plugins
func (r *Registry) ListPlugins(includeUnhealthy bool) []*ConnectedPlugin {
r.mu.RLock()
defer r.mu.RUnlock()
var result []*ConnectedPlugin
for _, plugin := range r.plugins {
if !includeUnhealthy && time.Since(plugin.LastHeartbeat) > r.healthCheckTimeout {
continue
}
result = append(result, plugin)
}
return result
}
// HealthCheck verifies plugin health based on heartbeat status
func (r *Registry) HealthCheck(pluginID string) (bool, error) {
r.mu.RLock()
plugin, exists := r.plugins[pluginID]
r.mu.RUnlock()
if !exists {
return false, fmt.Errorf("plugin %s not found", pluginID)
}
isHealthy := plugin.IsHealthy(r.healthCheckTimeout)
if !isHealthy {
r.mu.Lock()
r.pluginFailureCount[pluginID]++
r.mu.Unlock()
} else {
r.mu.Lock()
r.pluginFailureCount[pluginID] = 0
r.mu.Unlock()
}
return isHealthy, nil
}
// GetFailureCount returns the current failure count for a plugin
func (r *Registry) GetFailureCount(pluginID string) int {
r.mu.RLock()
defer r.mu.RUnlock()
return r.pluginFailureCount[pluginID]
}
// HasExceededFailureThreshold checks if a plugin has exceeded the failure threshold
func (r *Registry) HasExceededFailureThreshold(pluginID string) bool {
r.mu.RLock()
defer r.mu.RUnlock()
return r.pluginFailureCount[pluginID] > r.failureThreshold
}
// ResetFailureCount resets the failure counter for a plugin
func (r *Registry) ResetFailureCount(pluginID string) {
r.mu.Lock()
defer r.mu.Unlock()
r.pluginFailureCount[pluginID] = 0
}
// Count returns the total number of registered plugins
func (r *Registry) Count() int {
r.mu.RLock()
defer r.mu.RUnlock()
return len(r.plugins)
}
// CountHealthy returns the number of healthy plugins
func (r *Registry) CountHealthy() int {
r.mu.RLock()
defer r.mu.RUnlock()
count := 0
for _, plugin := range r.plugins {
if plugin.IsHealthy(r.healthCheckTimeout) {
count++
}
}
return count
}
// GetCapabilities returns all registered capabilities
func (r *Registry) GetCapabilities() []string {
r.mu.RLock()
defer r.mu.RUnlock()
var capabilities []string
for cap := range r.capabilityIndex {
capabilities = append(capabilities, cap)
}
return capabilities
}
// UpdateHeartbeat updates the heartbeat timestamp for a plugin
func (r *Registry) UpdateHeartbeat(pluginID string) error {
r.mu.RLock()
plugin, exists := r.plugins[pluginID]
r.mu.RUnlock()
if !exists {
return fmt.Errorf("plugin %s not found", pluginID)
}
plugin.UpdateHeartbeat()
r.ResetFailureCount(pluginID)
return nil
}
// GetUnhealthyPlugins returns plugins that have failed health checks
func (r *Registry) GetUnhealthyPlugins() []*ConnectedPlugin {
r.mu.RLock()
defer r.mu.RUnlock()
var unhealthy []*ConnectedPlugin
for _, plugin := range r.plugins {
if !plugin.IsHealthy(r.healthCheckTimeout) {
unhealthy = append(unhealthy, plugin)
}
}
return unhealthy
}
// RemoveUnhealthyPlugins removes plugins that have exceeded the failure threshold
func (r *Registry) RemoveUnhealthyPlugins() []string {
r.mu.Lock()
defer r.mu.Unlock()
var removed []string
for pluginID, failureCount := range r.pluginFailureCount {
if failureCount > r.failureThreshold {
if plugin, exists := r.plugins[pluginID]; exists {
// Remove from capability index
for _, cap := range plugin.Capabilities {
for i, id := range r.capabilityIndex[cap] {
if id == pluginID {
r.capabilityIndex[cap] = append(r.capabilityIndex[cap][:i], r.capabilityIndex[cap][i+1:]...)
break
}
}
}
delete(r.plugins, pluginID)
delete(r.pluginFailureCount, pluginID)
removed = append(removed, pluginID)
}
}
}
return removed
}
// UpdatePluginStatus updates the status field of a plugin
func (r *Registry) UpdatePluginStatus(pluginID, status string) error {
r.mu.RLock()
plugin, exists := r.plugins[pluginID]
r.mu.RUnlock()
if !exists {
return fmt.Errorf("plugin %s not found", pluginID)
}
plugin.mu.Lock()
plugin.Status = status
plugin.mu.Unlock()
return nil
}
// GetStats returns statistics for all plugins
func (r *Registry) GetStats() map[string]interface{} {
r.mu.RLock()
defer r.mu.RUnlock()
totalPlugins := len(r.plugins)
healthyPlugins := 0
totalActiveJobs := 0
totalCompletedJobs := 0
totalFailedJobs := 0
for _, plugin := range r.plugins {
if plugin.IsHealthy(r.healthCheckTimeout) {
healthyPlugins++
}
plugin.mu.RLock()
totalActiveJobs += plugin.ActiveJobs
totalCompletedJobs += plugin.CompletedJobs
totalFailedJobs += plugin.FailedJobs
plugin.mu.RUnlock()
}
return map[string]interface{}{
"total_plugins": totalPlugins,
"healthy_plugins": healthyPlugins,
"unhealthy_plugins": totalPlugins - healthyPlugins,
"total_active_jobs": totalActiveJobs,
"total_completed": totalCompletedJobs,
"total_failed": totalFailedJobs,
"capabilities": len(r.capabilityIndex),
}
}
-6
View File
@@ -1,6 +0,0 @@
package testing
import "errors"
// ErrSimulatedError is returned when error simulation is enabled
var ErrSimulatedError = errors.New("simulated plugin error")
-530
View File
@@ -1,530 +0,0 @@
package testing
import (
"context"
"fmt"
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// TestHarness provides a complete testing environment for plugins
type TestHarness struct {
mu sync.RWMutex
adminService *MockPluginService
plugins map[string]*MockPlugin
registrations map[string]*RegistrationRecord
jobs map[string]*JobTracker
detections map[string][]*DetectionRecord
executions map[string]*ExecutionRecord
startTime time.Time
timeout time.Duration
testName string
registrationWait time.Duration
executionWait time.Duration
expectedPlugins int
registeredPlugins int
failureReasons []string
}
// RegistrationRecord tracks plugin registration details
type RegistrationRecord struct {
PluginID string
RegisteredAt time.Time
Version string
Capabilities []string
MaxConcurrentJobs int
Status string
}
// JobTracker tracks job lifecycle
type JobTracker struct {
JobID string
Type string
PluginID string
Status plugin_pb.ExecutionStatus
CreatedAt time.Time
StartedAt *time.Time
CompletedAt *time.Time
Result *plugin_pb.JobResult
ErrorMessage string
Detections []*DetectionRecord
}
// DetectionRecord represents a detection result
type DetectionRecord struct {
ResourceID string
DetectionType string
Severity string
Description string
Data []byte
}
// ExecutionRecord tracks execution details
type ExecutionRecord struct {
ResourceID string
Type string
ExecutedAt time.Time
CompletedAt *time.Time
Success bool
ErrorMessage string
Data []byte
}
// NewTestHarness creates a new test harness
func NewTestHarness(testName string) *TestHarness {
return &TestHarness{
testName: testName,
adminService: NewMockPluginService(),
plugins: make(map[string]*MockPlugin),
registrations: make(map[string]*RegistrationRecord),
jobs: make(map[string]*JobTracker),
detections: make(map[string][]*DetectionRecord),
executions: make(map[string]*ExecutionRecord),
startTime: time.Now(),
timeout: 10 * time.Second,
registrationWait: 100 * time.Millisecond,
executionWait: 100 * time.Millisecond,
failureReasons: make([]string, 0),
}
}
// SetTimeout sets the overall test timeout
func (h *TestHarness) SetTimeout(timeout time.Duration) {
h.mu.Lock()
defer h.mu.Unlock()
h.timeout = timeout
}
// SetRegistrationWait sets the wait time for plugin registration
func (h *TestHarness) SetRegistrationWait(duration time.Duration) {
h.mu.Lock()
defer h.mu.Unlock()
h.registrationWait = duration
}
// SetExecutionWait sets the wait time for job execution
func (h *TestHarness) SetExecutionWait(duration time.Duration) {
h.mu.Lock()
defer h.mu.Unlock()
h.executionWait = duration
}
// RegisterPlugin simulates plugin registration
func (h *TestHarness) RegisterPlugin(plugin *MockPlugin) error {
h.mu.Lock()
if plugin == nil {
h.failureReasons = append(h.failureReasons, "plugin is nil")
h.mu.Unlock()
return fmt.Errorf("plugin is nil")
}
h.plugins[plugin.ID] = plugin
h.mu.Unlock()
// Simulate registration with admin service
req := &plugin_pb.PluginConnectRequest{
PluginId: plugin.ID,
PluginName: plugin.Name,
Version: plugin.Version,
Capabilities: plugin.Capabilities,
CapabilitiesDetail: plugin.CapabilitiesDetail,
MaxConcurrentJobs: int32(plugin.MaxConcurrentJobs),
}
ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
defer cancel()
resp, err := h.adminService.Connect(ctx, req)
if err != nil {
h.mu.Lock()
h.failureReasons = append(h.failureReasons, fmt.Sprintf("registration failed: %v", err))
h.mu.Unlock()
return err
}
if !resp.Success {
h.mu.Lock()
h.failureReasons = append(h.failureReasons, "registration response was not successful")
h.mu.Unlock()
return fmt.Errorf("registration failed: %s", resp.Message)
}
h.mu.Lock()
h.registrations[plugin.ID] = &RegistrationRecord{
PluginID: plugin.ID,
RegisteredAt: time.Now(),
Version: plugin.Version,
Capabilities: plugin.Capabilities,
MaxConcurrentJobs: plugin.MaxConcurrentJobs,
Status: "registered",
}
h.registeredPlugins++
h.mu.Unlock()
return nil
}
// RegisterMultiplePlugins registers multiple plugins
func (h *TestHarness) RegisterMultiplePlugins(plugins ...*MockPlugin) error {
for _, plugin := range plugins {
if err := h.RegisterPlugin(plugin); err != nil {
return err
}
}
return nil
}
// ExpectPlugins sets the expected number of plugins
func (h *TestHarness) ExpectPlugins(count int) {
h.mu.Lock()
defer h.mu.Unlock()
h.expectedPlugins = count
}
// DispatchJob sends a job to a plugin
func (h *TestHarness) DispatchJob(pluginID string, jobType string, payload *plugin_pb.JobPayload) (string, error) {
h.mu.RLock()
plugin, ok := h.plugins[pluginID]
h.mu.RUnlock()
if !ok {
return "", fmt.Errorf("plugin not found: %s", pluginID)
}
jobID := fmt.Sprintf("job-%d-%d", len(h.jobs), time.Now().UnixNano())
req := &plugin_pb.ExecuteJobRequest{
JobId: jobID,
JobType: jobType,
Payload: payload,
}
ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
defer cancel()
// Simulate job dispatch
err := h.adminService.SimulateJobExecution(req)
if err != nil {
h.mu.Lock()
h.failureReasons = append(h.failureReasons, fmt.Sprintf("job dispatch failed: %v", err))
h.mu.Unlock()
return "", err
}
// Wait for job to complete
time.Sleep(h.executionWait)
// Verify job execution
plugin.TrackJob(req)
_, executionErr := plugin.ExecuteJob(ctx, jobID, jobType, payload)
h.mu.Lock()
h.jobs[jobID] = &JobTracker{
JobID: jobID,
Type: jobType,
PluginID: pluginID,
Status: plugin_pb.ExecutionStatus_EXECUTION_STATUS_RUNNING,
CreatedAt: time.Now(),
}
h.mu.Unlock()
// Simulate completion after a small delay
time.Sleep(h.executionWait)
h.mu.Lock()
h.jobs[jobID].Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED
now := time.Now()
h.jobs[jobID].CompletedAt = &now
h.mu.Unlock()
if executionErr != nil {
h.mu.Lock()
h.jobs[jobID].Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED
h.jobs[jobID].ErrorMessage = executionErr.Error()
h.mu.Unlock()
}
return jobID, nil
}
// VerifyRegistration checks if a plugin was registered
func (h *TestHarness) VerifyRegistration(pluginID string) bool {
h.mu.RLock()
defer h.mu.RUnlock()
_, ok := h.registrations[pluginID]
return ok
}
// VerifyJobCompleted checks if a job completed successfully
func (h *TestHarness) VerifyJobCompleted(jobID string) bool {
h.mu.RLock()
defer h.mu.RUnlock()
job, ok := h.jobs[jobID]
if !ok {
return false
}
return job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED
}
// VerifyJobFailed checks if a job failed
func (h *TestHarness) VerifyJobFailed(jobID string) bool {
h.mu.RLock()
defer h.mu.RUnlock()
job, ok := h.jobs[jobID]
if !ok {
return false
}
return job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED
}
// VerifyPluginCapability checks if a plugin has a capability
func (h *TestHarness) VerifyPluginCapability(pluginID string, capability string) bool {
h.mu.RLock()
defer h.mu.RUnlock()
reg, ok := h.registrations[pluginID]
if !ok {
return false
}
for _, cap := range reg.Capabilities {
if cap == capability {
return true
}
}
return false
}
// GetJobStatus returns the status of a job
func (h *TestHarness) GetJobStatus(jobID string) plugin_pb.ExecutionStatus {
h.mu.RLock()
defer h.mu.RUnlock()
job, ok := h.jobs[jobID]
if !ok {
return plugin_pb.ExecutionStatus_EXECUTION_STATUS_UNKNOWN
}
return job.Status
}
// GetPlugin returns a registered plugin
func (h *TestHarness) GetPlugin(pluginID string) *MockPlugin {
h.mu.RLock()
defer h.mu.RUnlock()
return h.plugins[pluginID]
}
// GetRegistrationCount returns the number of registered plugins
func (h *TestHarness) GetRegistrationCount() int {
h.mu.RLock()
defer h.mu.RUnlock()
return h.registeredPlugins
}
// GetJobCount returns the total number of jobs dispatched
func (h *TestHarness) GetJobCount() int {
h.mu.RLock()
defer h.mu.RUnlock()
return len(h.jobs)
}
// SimulateDetection simulates detection results
func (h *TestHarness) SimulateDetection(pluginID string, result *DetectionRecord) error {
h.mu.RLock()
plugin, ok := h.plugins[pluginID]
h.mu.RUnlock()
if !ok {
return fmt.Errorf("plugin not found: %s", pluginID)
}
plugin.AddDetectionResult(result.ResourceID, result.DetectionType, result.Severity, result.Description, result.Data)
h.mu.Lock()
if _, exists := h.detections[pluginID]; !exists {
h.detections[pluginID] = make([]*DetectionRecord, 0)
}
h.detections[pluginID] = append(h.detections[pluginID], result)
h.mu.Unlock()
return nil
}
// GetAdminService returns the underlying admin service
func (h *TestHarness) GetAdminService() *MockPluginService {
h.mu.RLock()
defer h.mu.RUnlock()
return h.adminService
}
// GetTestDuration returns the elapsed test time
func (h *TestHarness) GetTestDuration() time.Duration {
h.mu.RLock()
defer h.mu.RUnlock()
return time.Since(h.startTime)
}
// ReportFailure records a test failure reason
func (h *TestHarness) ReportFailure(reason string) {
h.mu.Lock()
defer h.mu.Unlock()
h.failureReasons = append(h.failureReasons, reason)
}
// HasFailures checks if any failures were recorded
func (h *TestHarness) HasFailures() bool {
h.mu.RLock()
defer h.mu.RUnlock()
return len(h.failureReasons) > 0
}
// GetFailures returns all recorded failures
func (h *TestHarness) GetFailures() []string {
h.mu.RLock()
defer h.mu.RUnlock()
failures := make([]string, len(h.failureReasons))
copy(failures, h.failureReasons)
return failures
}
// WaitForRegistration waits for a specific number of plugins to register
func (h *TestHarness) WaitForRegistration(count int, timeout time.Duration) bool {
deadline := time.Now().Add(timeout)
for {
h.mu.RLock()
current := h.registeredPlugins
h.mu.RUnlock()
if current >= count {
return true
}
if time.Now().After(deadline) {
return false
}
time.Sleep(10 * time.Millisecond)
}
}
// VerifyAdminServiceStats checks admin service statistics
func (h *TestHarness) VerifyAdminServiceStats(regCount, jobCount int) bool {
return h.adminService.GetRegistrationCount() == regCount &&
h.adminService.GetJobDispatchCount() == jobCount
}
// GetCompletedJobCount returns the number of completed jobs
func (h *TestHarness) GetCompletedJobCount() int {
h.mu.RLock()
defer h.mu.RUnlock()
count := 0
for _, job := range h.jobs {
if job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED {
count++
}
}
return count
}
// GetFailedJobCount returns the number of failed jobs
func (h *TestHarness) GetFailedJobCount() int {
h.mu.RLock()
defer h.mu.RUnlock()
count := 0
for _, job := range h.jobs {
if job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED {
count++
}
}
return count
}
// Cleanup performs cleanup after a test
func (h *TestHarness) Cleanup() {
h.mu.Lock()
defer h.mu.Unlock()
// Reset all plugins
for _, plugin := range h.plugins {
plugin.Reset()
}
// Reset admin service
h.adminService.ResetCounters()
// Clear tracking
h.registrations = make(map[string]*RegistrationRecord)
h.jobs = make(map[string]*JobTracker)
h.detections = make(map[string][]*DetectionRecord)
h.executions = make(map[string]*ExecutionRecord)
h.failureReasons = make([]string, 0)
h.registeredPlugins = 0
h.startTime = time.Now()
}
// MockExecuteJobStream is a mock implementation of the ExecuteJob stream
type MockExecuteJobStream struct {
responses []*plugin_pb.ExecuteJobResponse
mu sync.Mutex
}
// Send sends a response on the stream
func (m *MockExecuteJobStream) Send(resp *plugin_pb.ExecuteJobResponse) error {
m.mu.Lock()
defer m.mu.Unlock()
m.responses = append(m.responses, resp)
return nil
}
// Recv receives a response from the stream
func (m *MockExecuteJobStream) Recv() (*plugin_pb.ExecuteJobResponse, error) {
m.mu.Lock()
defer m.mu.Unlock()
if len(m.responses) == 0 {
return nil, fmt.Errorf("no responses")
}
resp := m.responses[0]
m.responses = m.responses[1:]
return resp, nil
}
// SetHeader sets the metadata header
func (m *MockExecuteJobStream) SetHeader(map[string][]string) error {
return nil
}
// SendHeader sends the metadata header
func (m *MockExecuteJobStream) SendHeader(map[string][]string) error {
return nil
}
// SetTrailer sets the metadata trailer
func (m *MockExecuteJobStream) SetTrailer(map[string][]string) {
}
// Context returns the context
func (m *MockExecuteJobStream) Context() context.Context {
return context.Background()
}
// SendMsg sends a message on the stream
func (m *MockExecuteJobStream) SendMsg(interface{}) error {
return nil
}
// RecvMsg receives a message from the stream
func (m *MockExecuteJobStream) RecvMsg(interface{}) error {
return nil
}
-342
View File
@@ -1,342 +0,0 @@
package testing
import (
"context"
"sync"
"time"
"google.golang.org/protobuf/types/known/durationpb"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// MockPluginService simulates the admin-side PluginService for testing
type MockPluginService struct {
mu sync.RWMutex
plugins map[string]*MockPluginInstance
jobs map[string]*MockJob
jobCounter int
heartbeats map[string]*plugin_pb.HealthReport
lastHeartbeatTime map[string]time.Time
jobDispatchCalls int
registrationCalls int
receivedHealthReports []plugin_pb.HealthReport
}
// MockPluginInstance tracks a registered plugin
type MockPluginInstance struct {
ID string
Name string
Version string
Status string
Capabilities []string
MaxConcurrentJobs int
ConnectedAt time.Time
LastHeartbeat time.Time
ActiveJobCount int
CompletedJobCount int
FailedJobCount int
CapabilitiesDetail *plugin_pb.PluginCapabilities
Metadata map[string]string
}
// MockJob represents a job dispatched to a plugin
type MockJob struct {
ID string
Type string
PluginID string
Payload *plugin_pb.JobPayload
Timeout time.Duration
RetryCount int
Context map[string]string
Status plugin_pb.ExecutionStatus
DispatchedAt time.Time
ExecutedAt *time.Time
Result *plugin_pb.JobResult
ResultMessage string
StreamCalls int
}
// NewMockPluginService creates a new mock admin service
func NewMockPluginService() *MockPluginService {
return &MockPluginService{
plugins: make(map[string]*MockPluginInstance),
jobs: make(map[string]*MockJob),
heartbeats: make(map[string]*plugin_pb.HealthReport),
lastHeartbeatTime: make(map[string]time.Time),
receivedHealthReports: make([]plugin_pb.HealthReport, 0),
}
}
// Connect handles plugin registration
func (m *MockPluginService) Connect(ctx context.Context, req *plugin_pb.PluginConnectRequest) (*plugin_pb.PluginConnectResponse, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.registrationCalls++
// Register the plugin
plugin := &MockPluginInstance{
ID: req.PluginId,
Name: req.PluginName,
Version: req.Version,
Status: "connected",
Capabilities: req.Capabilities,
MaxConcurrentJobs: int(req.MaxConcurrentJobs),
ConnectedAt: time.Now(),
LastHeartbeat: time.Now(),
CapabilitiesDetail: req.CapabilitiesDetail,
Metadata: req.Metadata,
}
m.plugins[req.PluginId] = plugin
m.lastHeartbeatTime[req.PluginId] = time.Now()
// Build response with assigned types
assignedTypes := req.Capabilities
config := &plugin_pb.PluginConfig{
PluginId: req.PluginId,
Properties: make(map[string]string),
JobTypes: make([]*plugin_pb.JobTypeConfig, 0),
}
return &plugin_pb.PluginConnectResponse{
Success: true,
Message: "Plugin registered successfully",
MasterId: "mock-master-001",
AssignedTypes: assignedTypes,
Config: config,
}, nil
}
// SimulateJobExecution simulates job execution
func (m *MockPluginService) SimulateJobExecution(req *plugin_pb.ExecuteJobRequest) error {
m.mu.Lock()
m.jobDispatchCalls++
// Create job entry
job := &MockJob{
ID: req.JobId,
Type: req.JobType,
Payload: req.Payload,
Timeout: durationFromProto(req.Timeout),
RetryCount: int(req.RetryCount),
Context: req.Context,
DispatchedAt: time.Now(),
Status: plugin_pb.ExecutionStatus_EXECUTION_STATUS_ACCEPTED,
StreamCalls: 0,
}
m.jobs[req.JobId] = job
m.mu.Unlock()
// Simulate job execution
time.Sleep(50 * time.Millisecond)
// Update job status
m.mu.Lock()
job.StreamCalls++
job.Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_RUNNING
m.mu.Unlock()
// Simulate processing
time.Sleep(50 * time.Millisecond)
m.mu.Lock()
job.StreamCalls++
job.Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED
job.ResultMessage = "Job completed successfully"
now := time.Now()
job.ExecutedAt = &now
m.mu.Unlock()
return nil
}
// ExecuteJob simulates job dispatch
func (m *MockPluginService) ExecuteJob(ctx context.Context, req *plugin_pb.ExecuteJobRequest) (*plugin_pb.ExecuteJobResponse, error) {
m.mu.Lock()
m.jobDispatchCalls++
m.mu.Unlock()
return &plugin_pb.ExecuteJobResponse{
JobId: req.JobId,
Status: plugin_pb.ExecutionStatus_EXECUTION_STATUS_ACCEPTED,
Message: "Job accepted",
}, nil
}
// ReportHealth handles plugin health reports
func (m *MockPluginService) ReportHealth(ctx context.Context, report *plugin_pb.HealthReport) (*plugin_pb.HealthReportResponse, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.heartbeats[report.PluginId] = report
m.lastHeartbeatTime[report.PluginId] = time.Now()
m.receivedHealthReports = append(m.receivedHealthReports, *report)
// Update plugin status
if plugin, ok := m.plugins[report.PluginId]; ok {
plugin.LastHeartbeat = time.Now()
plugin.ActiveJobCount = int(report.ActiveJobs)
}
return &plugin_pb.HealthReportResponse{
Acknowledged: true,
Feedback: "Health report received",
}, nil
}
// GetConfig handles config retrieval
func (m *MockPluginService) GetConfig(ctx context.Context, req *plugin_pb.GetConfigRequest) (*plugin_pb.GetConfigResponse, error) {
m.mu.RLock()
defer m.mu.RUnlock()
config := &plugin_pb.PluginConfig{
PluginId: req.PluginId,
Properties: make(map[string]string),
JobTypes: make([]*plugin_pb.JobTypeConfig, 0),
}
return &plugin_pb.GetConfigResponse{
Config: config,
Version: 1,
}, nil
}
// SubmitResult handles job result submission
func (m *MockPluginService) SubmitResult(ctx context.Context, req *plugin_pb.JobResultRequest) (*plugin_pb.JobResultResponse, error) {
m.mu.Lock()
defer m.mu.Unlock()
if job, ok := m.jobs[req.JobId]; ok {
job.Status = req.Status
job.Result = req.Result
job.ResultMessage = req.Message
}
return &plugin_pb.JobResultResponse{
Acknowledged: true,
ActionsToTake: []string{},
}, nil
}
// GetRegistrationCount returns how many times Connect was called
func (m *MockPluginService) GetRegistrationCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return m.registrationCalls
}
// GetJobDispatchCount returns how many times ExecuteJob was called
func (m *MockPluginService) GetJobDispatchCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return m.jobDispatchCalls
}
// GetPluginCount returns the number of registered plugins
func (m *MockPluginService) GetPluginCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return len(m.plugins)
}
// GetPlugin returns a registered plugin by ID
func (m *MockPluginService) GetPlugin(pluginID string) *MockPluginInstance {
m.mu.RLock()
defer m.mu.RUnlock()
return m.plugins[pluginID]
}
// GetJob returns a dispatched job by ID
func (m *MockPluginService) GetJob(jobID string) *MockJob {
m.mu.RLock()
defer m.mu.RUnlock()
return m.jobs[jobID]
}
// GetJobCount returns the total number of dispatched jobs
func (m *MockPluginService) GetJobCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return len(m.jobs)
}
// GetLastHeartbeat returns the last heartbeat time for a plugin
func (m *MockPluginService) GetLastHeartbeat(pluginID string) *time.Time {
m.mu.RLock()
defer m.mu.RUnlock()
if t, ok := m.lastHeartbeatTime[pluginID]; ok {
return &t
}
return nil
}
// GetHeartbeatCount returns how many heartbeats have been received
func (m *MockPluginService) GetHeartbeatCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return len(m.receivedHealthReports)
}
// ResetCounters resets all counters for a fresh test
func (m *MockPluginService) ResetCounters() {
m.mu.Lock()
defer m.mu.Unlock()
m.registrationCalls = 0
m.jobDispatchCalls = 0
m.plugins = make(map[string]*MockPluginInstance)
m.jobs = make(map[string]*MockJob)
m.heartbeats = make(map[string]*plugin_pb.HealthReport)
m.lastHeartbeatTime = make(map[string]time.Time)
m.receivedHealthReports = make([]plugin_pb.HealthReport, 0)
}
// VerifyJobCompleted checks if a job was completed successfully
func (m *MockPluginService) VerifyJobCompleted(jobID string) bool {
m.mu.RLock()
defer m.mu.RUnlock()
job, ok := m.jobs[jobID]
if !ok {
return false
}
return job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED
}
// VerifyJobFailed checks if a job failed
func (m *MockPluginService) VerifyJobFailed(jobID string) bool {
m.mu.RLock()
defer m.mu.RUnlock()
job, ok := m.jobs[jobID]
if !ok {
return false
}
return job.Status == plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED
}
// GetJobStatus returns the current status of a job
func (m *MockPluginService) GetJobStatus(jobID string) plugin_pb.ExecutionStatus {
m.mu.RLock()
defer m.mu.RUnlock()
if job, ok := m.jobs[jobID]; ok {
return job.Status
}
return plugin_pb.ExecutionStatus_EXECUTION_STATUS_UNKNOWN
}
// VerifyPluginRegistered checks if a plugin is registered
func (m *MockPluginService) VerifyPluginRegistered(pluginID string) bool {
m.mu.RLock()
defer m.mu.RUnlock()
_, ok := m.plugins[pluginID]
return ok
}
// durationFromProto converts proto Duration to time.Duration
func durationFromProto(d *durationpb.Duration) time.Duration {
if d == nil {
return 0
}
return time.Duration(d.Seconds)*time.Second + time.Duration(d.Nanos)
}
-449
View File
@@ -1,449 +0,0 @@
package testing
import (
"context"
"sync"
"time"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// MockPlugin simulates a plugin worker instance for testing
type MockPlugin struct {
mu sync.RWMutex
ID string
Name string
Version string
Status string
Capabilities []string
CapabilitiesDetail *plugin_pb.PluginCapabilities
MaxConcurrentJobs int
Config *plugin_pb.PluginConfig
ActiveJobs map[string]*MockJobExecution
CompletedJobs int
FailedJobs int
ConnectStreamCalls int
ExecuteJobStreamCalls int
ReportHealthCalls int
GetConfigCalls int
SubmitResultCalls int
ReceivedJobs []*plugin_pb.ExecuteJobRequest
ReceivedHealthReports []*plugin_pb.HealthReport
LastError string
SimulateError bool
SimulateErrorType string
SchemaData []byte
DetectionResults []*DetectionResult
ExecutionResults []*ExecutionResult
}
// MockJobExecution tracks job execution state
type MockJobExecution struct {
JobID string
Type string
Status plugin_pb.ExecutionStatus
Progress float32
CurrentStep string
StartTime time.Time
EndTime *time.Time
Result *plugin_pb.JobResult
ErrorMessage string
}
// DetectionResult represents detection results
type DetectionResult struct {
ResourceID string
DetectionType string
Severity string
Description string
Data []byte
}
// ExecutionResult represents execution results
type ExecutionResult struct {
ResourceID string
Success bool
ErrorMessage string
Data []byte
}
// NewMockPlugin creates a new mock plugin
func NewMockPlugin(id, name, version string) *MockPlugin {
return &MockPlugin{
ID: id,
Name: name,
Version: version,
Status: "ready",
Capabilities: make([]string, 0),
CapabilitiesDetail: &plugin_pb.PluginCapabilities{},
MaxConcurrentJobs: 5,
Config: &plugin_pb.PluginConfig{},
ActiveJobs: make(map[string]*MockJobExecution),
ReceivedJobs: make([]*plugin_pb.ExecuteJobRequest, 0),
ReceivedHealthReports: make([]*plugin_pb.HealthReport, 0),
DetectionResults: make([]*DetectionResult, 0),
ExecutionResults: make([]*ExecutionResult, 0),
}
}
// AddCapability adds a capability to the plugin
func (m *MockPlugin) AddCapability(cap string) {
m.mu.Lock()
defer m.mu.Unlock()
m.Capabilities = append(m.Capabilities, cap)
}
// AddDetectionCapability adds a detection capability
func (m *MockPlugin) AddDetectionCapability(typ, desc string, minInterval int32, requiresFullScan bool) {
m.mu.Lock()
defer m.mu.Unlock()
if m.CapabilitiesDetail == nil {
m.CapabilitiesDetail = &plugin_pb.PluginCapabilities{}
}
m.CapabilitiesDetail.Detection = append(m.CapabilitiesDetail.Detection, &plugin_pb.DetectionCapability{
Type: typ,
Description: desc,
MinIntervalSeconds: minInterval,
RequiresFullScan: requiresFullScan,
})
m.Capabilities = append(m.Capabilities, typ)
}
// AddMaintenanceCapability adds a maintenance capability
func (m *MockPlugin) AddMaintenanceCapability(typ, desc string, requiredDetections []string) {
m.mu.Lock()
defer m.mu.Unlock()
if m.CapabilitiesDetail == nil {
m.CapabilitiesDetail = &plugin_pb.PluginCapabilities{}
}
m.CapabilitiesDetail.Maintenance = append(m.CapabilitiesDetail.Maintenance, &plugin_pb.MaintenanceCapability{
Type: typ,
Description: desc,
RequiredDetectionTypes: requiredDetections,
})
}
// SetSchema sets the schema data
func (m *MockPlugin) SetSchema(data []byte) {
m.mu.Lock()
defer m.mu.Unlock()
m.SchemaData = data
}
// AddDetectionResult adds a detection result
func (m *MockPlugin) AddDetectionResult(resourceID, detectionType, severity, description string, data []byte) {
m.mu.Lock()
defer m.mu.Unlock()
m.DetectionResults = append(m.DetectionResults, &DetectionResult{
ResourceID: resourceID,
DetectionType: detectionType,
Severity: severity,
Description: description,
Data: data,
})
}
// AddExecutionResult adds an execution result
func (m *MockPlugin) AddExecutionResult(resourceID string, success bool, errorMsg string, data []byte) {
m.mu.Lock()
defer m.mu.Unlock()
m.ExecutionResults = append(m.ExecutionResults, &ExecutionResult{
ResourceID: resourceID,
Success: success,
ErrorMessage: errorMsg,
Data: data,
})
}
// GetConfigurationSchema implements schema retrieval
func (m *MockPlugin) GetConfigurationSchema(ctx context.Context) ([]byte, error) {
m.mu.RLock()
defer m.mu.RUnlock()
if m.SimulateError && m.SimulateErrorType == "schema" {
return nil, ErrSimulatedError
}
return m.SchemaData, nil
}
// DetectJobs implements detection logic
func (m *MockPlugin) DetectJobs(ctx context.Context) ([]*DetectionResult, error) {
m.mu.Lock()
m.ReportHealthCalls++
results := make([]*DetectionResult, len(m.DetectionResults))
copy(results, m.DetectionResults)
m.mu.Unlock()
if m.SimulateError && m.SimulateErrorType == "detect" {
return nil, ErrSimulatedError
}
return results, nil
}
// ExecuteJob implements job execution
func (m *MockPlugin) ExecuteJob(ctx context.Context, jobID string, jobType string, payload *plugin_pb.JobPayload) (*ExecutionResult, error) {
m.mu.Lock()
m.ExecuteJobStreamCalls++
execution := &MockJobExecution{
JobID: jobID,
Type: jobType,
Status: plugin_pb.ExecutionStatus_EXECUTION_STATUS_RUNNING,
StartTime: time.Now(),
Progress: 0,
CurrentStep: "initialized",
}
m.ActiveJobs[jobID] = execution
m.mu.Unlock()
// Simulate execution steps
steps := []string{"initialized", "validating", "processing", "finalizing"}
for i, step := range steps {
select {
case <-ctx.Done():
m.mu.Lock()
execution.Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_CANCELLED
execution.ErrorMessage = "context cancelled"
delete(m.ActiveJobs, jobID)
m.mu.Unlock()
return nil, ctx.Err()
default:
}
m.mu.Lock()
execution.CurrentStep = step
execution.Progress = float32((i + 1) * 25)
m.mu.Unlock()
time.Sleep(10 * time.Millisecond)
}
m.mu.Lock()
defer m.mu.Unlock()
if m.SimulateError && m.SimulateErrorType == "execute" {
execution.Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_FAILED
execution.ErrorMessage = "simulated execution error"
m.FailedJobs++
delete(m.ActiveJobs, jobID)
return nil, ErrSimulatedError
}
// Get results
result := &ExecutionResult{
ResourceID: jobID,
Success: true,
ErrorMessage: "",
}
if len(m.ExecutionResults) > 0 {
result = m.ExecutionResults[0]
m.ExecutionResults = m.ExecutionResults[1:]
}
execution.Status = plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED
execution.Progress = 100
execution.CurrentStep = "completed"
now := time.Now()
execution.EndTime = &now
m.CompletedJobs++
delete(m.ActiveJobs, jobID)
return result, nil
}
// ConnectStream simulates the Connect RPC stream
func (m *MockPlugin) ConnectStream(ctx context.Context, conn grpc.ClientConnInterface) error {
m.mu.Lock()
m.ConnectStreamCalls++
m.mu.Unlock()
if m.SimulateError && m.SimulateErrorType == "connect" {
return ErrSimulatedError
}
return nil
}
// ExecuteJobStream simulates the ExecuteJob RPC stream
func (m *MockPlugin) ExecuteJobStream(ctx context.Context, conn grpc.ClientConnInterface, jobID string) error {
m.mu.Lock()
m.ExecuteJobStreamCalls++
m.mu.Unlock()
if m.SimulateError && m.SimulateErrorType == "executestream" {
return ErrSimulatedError
}
return nil
}
// ReportHealth sends a health report
func (m *MockPlugin) ReportHealth(ctx context.Context, conn grpc.ClientConnInterface) error {
m.mu.Lock()
m.ReportHealthCalls++
activeCount := len(m.ActiveJobs)
m.mu.Unlock()
if m.SimulateError && m.SimulateErrorType == "health" {
return ErrSimulatedError
}
report := &plugin_pb.HealthReport{
PluginId: m.ID,
TimestampMs: time.Now().UnixMilli(),
Status: plugin_pb.HealthStatus_HEALTH_STATUS_HEALTHY,
ActiveJobs: int32(activeCount),
}
m.mu.Lock()
m.ReceivedHealthReports = append(m.ReceivedHealthReports, report)
m.mu.Unlock()
return nil
}
// GetConfig retrieves configuration
func (m *MockPlugin) GetConfig(ctx context.Context, conn grpc.ClientConnInterface) (*plugin_pb.PluginConfig, error) {
m.mu.Lock()
m.GetConfigCalls++
defer m.mu.Unlock()
if m.SimulateError && m.SimulateErrorType == "getconfig" {
return nil, ErrSimulatedError
}
return m.Config, nil
}
// SubmitResult submits job results
func (m *MockPlugin) SubmitResult(ctx context.Context, conn grpc.ClientConnInterface, jobID string, result *plugin_pb.JobResult) error {
m.mu.Lock()
m.SubmitResultCalls++
defer m.mu.Unlock()
if m.SimulateError && m.SimulateErrorType == "submitresult" {
return ErrSimulatedError
}
return nil
}
// GetActiveJobCount returns the number of active jobs
func (m *MockPlugin) GetActiveJobCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return len(m.ActiveJobs)
}
// GetCompletedJobCount returns the number of completed jobs
func (m *MockPlugin) GetCompletedJobCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return m.CompletedJobs
}
// GetFailedJobCount returns the number of failed jobs
func (m *MockPlugin) GetFailedJobCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return m.FailedJobs
}
// GetStreamCallCount returns the count of stream calls
func (m *MockPlugin) GetStreamCallCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return m.ExecuteJobStreamCalls
}
// GetHealthReportCount returns the count of health reports sent
func (m *MockPlugin) GetHealthReportCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return m.ReportHealthCalls
}
// EnableErrorSimulation enables error simulation
func (m *MockPlugin) EnableErrorSimulation(errorType string) {
m.mu.Lock()
defer m.mu.Unlock()
m.SimulateError = true
m.SimulateErrorType = errorType
}
// DisableErrorSimulation disables error simulation
func (m *MockPlugin) DisableErrorSimulation() {
m.mu.Lock()
defer m.mu.Unlock()
m.SimulateError = false
m.SimulateErrorType = ""
}
// Reset clears all counters and state
func (m *MockPlugin) Reset() {
m.mu.Lock()
defer m.mu.Unlock()
m.ActiveJobs = make(map[string]*MockJobExecution)
m.CompletedJobs = 0
m.FailedJobs = 0
m.ConnectStreamCalls = 0
m.ExecuteJobStreamCalls = 0
m.ReportHealthCalls = 0
m.GetConfigCalls = 0
m.SubmitResultCalls = 0
m.ReceivedJobs = make([]*plugin_pb.ExecuteJobRequest, 0)
m.ReceivedHealthReports = make([]*plugin_pb.HealthReport, 0)
m.LastError = ""
m.SimulateError = false
m.SimulateErrorType = ""
}
// GetJobExecution returns execution details for a job
func (m *MockPlugin) GetJobExecution(jobID string) *MockJobExecution {
m.mu.RLock()
defer m.mu.RUnlock()
return m.ActiveJobs[jobID]
}
// TrackJob records a received job
func (m *MockPlugin) TrackJob(req *plugin_pb.ExecuteJobRequest) {
m.mu.Lock()
defer m.mu.Unlock()
m.ReceivedJobs = append(m.ReceivedJobs, req)
}
// GetReceivedJobCount returns the count of received jobs
func (m *MockPlugin) GetReceivedJobCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return len(m.ReceivedJobs)
}
// SimulateStreamError simulates an error during streaming
func (m *MockPlugin) SimulateStreamError(reason error) {
m.mu.Lock()
defer m.mu.Unlock()
m.LastError = reason.Error()
}
// SetStatus sets the plugin status
func (m *MockPlugin) SetStatus(status string) {
m.mu.Lock()
defer m.mu.Unlock()
m.Status = status
}
// GetStatus returns the plugin status
func (m *MockPlugin) GetStatus() string {
m.mu.RLock()
defer m.mu.RUnlock()
return m.Status
}
-265
View File
@@ -1,265 +0,0 @@
package plugin
import (
"sync"
"time"
)
// JobState represents the current state of a job in the state machine
type JobState int
const (
JobStatePending JobState = iota
JobStateScheduled
JobStateRunning
JobStateCompleted
JobStateFailed
JobStateCancelled
)
func (s JobState) String() string {
switch s {
case JobStatePending:
return "PENDING"
case JobStateScheduled:
return "SCHEDULED"
case JobStateRunning:
return "RUNNING"
case JobStateCompleted:
return "COMPLETED"
case JobStateFailed:
return "FAILED"
case JobStateCancelled:
return "CANCELLED"
default:
return "UNKNOWN"
}
}
// Job represents a detection or maintenance task
type Job struct {
mu sync.RWMutex
ID string
Type string
PluginID string
State JobState
Payload interface{}
CreatedAt time.Time
StartedAt *time.Time
CompletedAt *time.Time
ExecutionTime time.Duration
RetryCount int
MaxRetries int
LastError string
Result *JobResult
DetectionRecords []DetectionRecord
}
// GetState safely retrieves the job state
func (j *Job) GetState() JobState {
j.mu.RLock()
defer j.mu.RUnlock()
return j.State
}
// SetState safely updates the job state
func (j *Job) SetState(state JobState) {
j.mu.Lock()
defer j.mu.Unlock()
j.State = state
if state == JobStateRunning {
now := time.Now()
j.StartedAt = &now
} else if state == JobStateCompleted || state == JobStateFailed || state == JobStateCancelled {
now := time.Now()
j.CompletedAt = &now
if j.StartedAt != nil {
j.ExecutionTime = j.CompletedAt.Sub(*j.StartedAt)
}
}
}
// JobResult contains the output from job execution
type JobResult struct {
Success bool
Data []byte
Warnings []string
Errors []string
Metadata map[string]string
Message string
}
// DetectionRecord represents a single detection result
type DetectionRecord struct {
DetectionType string
Timestamp time.Time
Severity string
Description string
AffectedResource string
RawData []byte
}
// ExecutionRecord persists job execution history
type ExecutionRecord struct {
JobID string
JobType string
PluginID string
State JobState
CreatedAt time.Time
StartedAt *time.Time
CompletedAt *time.Time
Payload interface{}
Result *JobResult
RetryCount int
LastError string
}
// ConnectedPlugin represents a connected plugin instance
type ConnectedPlugin struct {
mu sync.RWMutex
ID string
Name string
Version string
Status string
Capabilities []string
MaxConcurrentJobs int
ActiveJobs int
CompletedJobs int
FailedJobs int
TotalDetections int64
AvgExecutionTimeMs float64
CPUUsagePercent float64
MemoryUsageBytes int64
ConnectedAt time.Time
LastHeartbeat time.Time
Metadata map[string]string
HealthCheckInterval time.Duration
JobTimeout time.Duration
}
// IsHealthy checks if the plugin is considered healthy based on heartbeat
func (cp *ConnectedPlugin) IsHealthy(timeout time.Duration) bool {
cp.mu.RLock()
defer cp.mu.RUnlock()
return time.Since(cp.LastHeartbeat) < timeout
}
// UpdateHeartbeat updates the last heartbeat timestamp
func (cp *ConnectedPlugin) UpdateHeartbeat() {
cp.mu.Lock()
defer cp.mu.Unlock()
cp.LastHeartbeat = time.Now()
}
// IncActiveJobs increments active job counter
func (cp *ConnectedPlugin) IncActiveJobs() {
cp.mu.Lock()
defer cp.mu.Unlock()
cp.ActiveJobs++
}
// DecActiveJobs decrements active job counter
func (cp *ConnectedPlugin) DecActiveJobs() {
cp.mu.Lock()
defer cp.mu.Unlock()
if cp.ActiveJobs > 0 {
cp.ActiveJobs--
}
}
// JobTypeConfig defines configuration for a specific job type
type JobTypeConfig struct {
Type string
Enabled bool
Priority int
Interval time.Duration
MaxConcurrent int
Parameters map[string]string
RequiredDetections []string
DetectionHistory []DetectionRecord
ExecutionHistory []ExecutionRecord
}
// PluginConfig holds all configuration for a plugin
type PluginConfig struct {
mu sync.RWMutex
PluginID string
Properties map[string]string
JobTypes map[string]*JobTypeConfig
MaxRetries int
HealthCheckInterval time.Duration
JobTimeout time.Duration
Environment map[string]string
}
// GetProperty safely retrieves a configuration property
func (pc *PluginConfig) GetProperty(key string) (string, bool) {
pc.mu.RLock()
defer pc.mu.RUnlock()
val, ok := pc.Properties[key]
return val, ok
}
// SetProperty safely sets a configuration property
func (pc *PluginConfig) SetProperty(key, value string) {
pc.mu.Lock()
defer pc.mu.Unlock()
if pc.Properties == nil {
pc.Properties = make(map[string]string)
}
pc.Properties[key] = value
}
// GetJobTypeConfig safely retrieves job type configuration
func (pc *PluginConfig) GetJobTypeConfig(jobType string) (*JobTypeConfig, bool) {
pc.mu.RLock()
defer pc.mu.RUnlock()
cfg, ok := pc.JobTypes[jobType]
return cfg, ok
}
// SetJobTypeConfig safely sets job type configuration
func (pc *PluginConfig) SetJobTypeConfig(jobType string, cfg *JobTypeConfig) {
pc.mu.Lock()
defer pc.mu.Unlock()
if pc.JobTypes == nil {
pc.JobTypes = make(map[string]*JobTypeConfig)
}
pc.JobTypes[jobType] = cfg
}
// PluginHealth represents the health status of a plugin
type PluginHealth struct {
mu sync.RWMutex
PluginID string
Status string
ActiveJobs int
CPUPercent int64
MemoryBytes int64
Timestamp time.Time
JobProgressList []JobProgress
}
// JobProgress tracks progress of an executing job
type JobProgress struct {
JobID string
ProgressPercent float32
CurrentStep string
}
// DetectionCapability describes what a plugin can detect
type DetectionCapability struct {
Type string
Description string
MinIntervalSeconds int
RequiresFullScan bool
OutputMetrics []string
}
// MaintenanceCapability describes maintenance operations a plugin can perform
type MaintenanceCapability struct {
Type string
Description string
RequiredDetectionTypes []string
EstimatedDurationSeconds int
}
@@ -1,187 +0,0 @@
package balance
import (
"fmt"
"math"
)
// RebalanceCandidate represents a rebalance opportunity
type RebalanceCandidate struct {
VolumeID uint32
SourceNodeID string
DestinationNodeID string
SourceUsagePercent float32
DestinationUsagePercent float32
ImbalanceScore float32
DataToMove uint64
ExpectedBenefit float32
Priority int
CanExecute bool
Reason string
}
// DetectionOptions contains options for detection
type DetectionOptions struct {
AcceptableImbalance float32
DiskUsageThreshold float32
MinVolumeSize uint64
MaxVolumeSize uint64
PreferBalancedDist bool
PreferredNodes []string
ExcludeNodes []string
}
// Detector scans for rebalance opportunities
type Detector struct {
config DetectionOptions
}
// NewDetector creates a new balance detector
func NewDetector(opts DetectionOptions) *Detector {
return &Detector{
config: opts,
}
}
// DetectJobs analyzes disk usage and identifies rebalance opportunities
func (d *Detector) DetectJobs(nodeMetrics map[string]*NodeMetric) ([]*RebalanceCandidate, error) {
candidates := make([]*RebalanceCandidate, 0)
// Calculate cluster statistics
avgUsage, stdDev := d.calculateClusterStats(nodeMetrics)
// Find imbalanced nodes
for sourceID, sourceMetric := range nodeMetrics {
if d.isNodeExcluded(sourceID) {
continue
}
if sourceMetric.UsagePercent > avgUsage+stdDev {
// Source node is above average
for destID, destMetric := range nodeMetrics {
if sourceID == destID || d.isNodeExcluded(destID) {
continue
}
if destMetric.UsagePercent < avgUsage-stdDev {
// Found a destination below average
candidate := d.evaluateRebalanceOpportunity(
sourceID, sourceMetric,
destID, destMetric,
)
if candidate.CanExecute {
candidates = append(candidates, candidate)
}
}
}
}
}
SortByImbalance(candidates)
return candidates, nil
}
// evaluateRebalanceOpportunity evaluates a single rebalance opportunity
func (d *Detector) evaluateRebalanceOpportunity(
sourceID string, sourceMetric *NodeMetric,
destID string, destMetric *NodeMetric,
) *RebalanceCandidate {
candidate := &RebalanceCandidate{
SourceNodeID: sourceID,
DestinationNodeID: destID,
SourceUsagePercent: sourceMetric.UsagePercent,
DestinationUsagePercent: destMetric.UsagePercent,
}
// Check destination capacity
if !d.checkNodeCapacity(destMetric) {
candidate.CanExecute = false
candidate.Reason = "destination node insufficient free space"
return candidate
}
// Calculate imbalance score
imbalance := math.Abs(float64(sourceMetric.UsagePercent - destMetric.UsagePercent))
candidate.ImbalanceScore = float32(imbalance)
// Check if imbalance exceeds acceptable level
if candidate.ImbalanceScore < d.config.AcceptableImbalance {
candidate.CanExecute = false
candidate.Reason = fmt.Sprintf("imbalance below threshold: %.2f < %.2f", candidate.ImbalanceScore, d.config.AcceptableImbalance)
return candidate
}
// Calculate data to move (simplified)
candidate.DataToMove = uint64(sourceMetric.UsedSpace / 10)
candidate.ExpectedBenefit = candidate.ImbalanceScore / 2
candidate.CanExecute = true
candidate.Priority = int(candidate.ImbalanceScore)
candidate.Reason = "eligible for rebalancing"
return candidate
}
// checkNodeCapacity checks if destination node has sufficient capacity
func (d *Detector) checkNodeCapacity(metric *NodeMetric) bool {
freeSpacePercent := 100 - metric.UsagePercent
return freeSpacePercent > 20 // Need at least 20% free
}
// calculateClusterStats calculates average usage and standard deviation
func (d *Detector) calculateClusterStats(nodeMetrics map[string]*NodeMetric) (float32, float32) {
if len(nodeMetrics) == 0 {
return 0, 0
}
var sum float32
for _, metric := range nodeMetrics {
sum += metric.UsagePercent
}
avg := sum / float32(len(nodeMetrics))
var sumDiffSq float32
for _, metric := range nodeMetrics {
diff := metric.UsagePercent - avg
sumDiffSq += diff * diff
}
variance := sumDiffSq / float32(len(nodeMetrics))
stdDev := float32(math.Sqrt(float64(variance)))
return avg, stdDev
}
// isNodeExcluded checks if a node is in the exclusion list
func (d *Detector) isNodeExcluded(nodeID string) bool {
for _, excluded := range d.config.ExcludeNodes {
if excluded == nodeID {
return true
}
}
return false
}
// NodeMetric contains node statistics
type NodeMetric struct {
NodeID string
TotalSpace uint64
UsedSpace uint64
FreeSpace uint64
UsagePercent float32
VolumeCount int
LastUpdated int64
IsHealthy bool
}
// SortByImbalance sorts candidates by imbalance score
func SortByImbalance(candidates []*RebalanceCandidate) {
for i := 0; i < len(candidates); i++ {
for j := i + 1; j < len(candidates); j++ {
if candidates[j].ImbalanceScore > candidates[i].ImbalanceScore {
candidates[i], candidates[j] = candidates[j], candidates[i]
}
}
}
}
@@ -1,255 +0,0 @@
package balance
import (
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// ExecutionStatus tracks job execution status
type ExecutionStatus string
const (
StatusValidating ExecutionStatus = "validating"
StatusSelecting ExecutionStatus = "selecting"
StatusTransferring ExecutionStatus = "transferring"
StatusUpdating ExecutionStatus = "updating"
StatusVerifying ExecutionStatus = "verifying"
StatusCompleted ExecutionStatus = "completed"
StatusFailed ExecutionStatus = "failed"
)
// ExecutionStep represents a step in the rebalance pipeline
type ExecutionStep struct {
Name string
Status ExecutionStatus
StartTime *time.Time
EndTime *time.Time
Progress float32
ErrorMsg string
}
// Executor handles rebalance execution
type Executor struct {
config *ExecutorConfig
}
// ExecutorConfig contains executor configuration
type ExecutorConfig struct {
MinVolumeSize uint64
MaxVolumeSize uint64
TimeoutPerStep time.Duration
MaxRetries int
}
// NewExecutor creates a new balance executor
func NewExecutor(config *ExecutorConfig) *Executor {
if config == nil {
config = &ExecutorConfig{
MinVolumeSize: 500,
MaxVolumeSize: 10000,
TimeoutPerStep: 2 * time.Minute,
MaxRetries: 3,
}
}
return &Executor{config: config}
}
// BalanceExecutionResult contains the result of rebalance operation
type BalanceExecutionResult struct {
SourceNode string
DestinationNode string
Success bool
StartTime time.Time
EndTime time.Time
TotalDuration time.Duration
BytesTransferred uint64
VolumesMovedCount int
Metadata map[string]string
Steps []*ExecutionStep
ErrorMessage string
}
// ExecuteJob executes the rebalance operation
func (e *Executor) ExecuteJob(job *plugin_pb.ExecuteJobRequest, source, dest string) (*BalanceExecutionResult, error) {
result := &BalanceExecutionResult{
SourceNode: source,
DestinationNode: dest,
Success: false,
StartTime: time.Now(),
Metadata: make(map[string]string),
Steps: make([]*ExecutionStep, 0),
}
// Step 1: Validate balance state
if err := e.validateBalance(result); err != nil {
result.ErrorMessage = fmt.Sprintf("validation failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 2: Select volume to move
if err := e.selectVolume(result); err != nil {
result.ErrorMessage = fmt.Sprintf("selection failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 3: Transfer data
if err := e.transferData(result); err != nil {
result.ErrorMessage = fmt.Sprintf("transfer failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 4: Update mapping
if err := e.updateMapping(result); err != nil {
result.ErrorMessage = fmt.Sprintf("mapping update failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 5: Verify balance
if err := e.verifyBalance(result); err != nil {
result.ErrorMessage = fmt.Sprintf("verification failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
result.Success = true
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, nil
}
// validateBalance validates current balance state
func (e *Executor) validateBalance(result *BalanceExecutionResult) error {
step := &ExecutionStep{
Name: "validating",
Status: StatusValidating,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
time.Sleep(50 * time.Millisecond)
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// selectVolume selects a volume to move
func (e *Executor) selectVolume(result *BalanceExecutionResult) error {
step := &ExecutionStep{
Name: "selecting",
Status: StatusSelecting,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
time.Sleep(30 * time.Millisecond)
result.VolumesMovedCount = 1
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// transferData transfers data to destination
func (e *Executor) transferData(result *BalanceExecutionResult) error {
step := &ExecutionStep{
Name: "transferring",
Status: StatusTransferring,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
for i := 0; i < 10; i++ {
time.Sleep(40 * time.Millisecond)
step.Progress = float32((i + 1) * 10)
}
result.BytesTransferred = 500000
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// updateMapping updates volume mapping
func (e *Executor) updateMapping(result *BalanceExecutionResult) error {
step := &ExecutionStep{
Name: "updating",
Status: StatusUpdating,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
time.Sleep(50 * time.Millisecond)
result.Metadata["source_usage_before"] = "80%"
result.Metadata["dest_usage_before"] = "40%"
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// verifyBalance verifies the new balance state
func (e *Executor) verifyBalance(result *BalanceExecutionResult) error {
step := &ExecutionStep{
Name: "verifying",
Status: StatusVerifying,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
time.Sleep(50 * time.Millisecond)
result.Metadata["source_usage_after"] = "76%"
result.Metadata["dest_usage_after"] = "44%"
result.Metadata["imbalance_reduction"] = "8%"
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// ValidateExecutionResult validates the result of execution
func ValidateExecutionResult(result *BalanceExecutionResult) bool {
if !result.Success {
return false
}
if result.EndTime.Before(result.StartTime) {
return false
}
if len(result.Steps) != 5 {
return false
}
return true
}
-200
View File
@@ -1,200 +0,0 @@
package balance
import (
"encoding/json"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// ConfigurationSchema defines the schema for balance plugin configuration
type ConfigurationSchema struct {
AdminConfig AdminConfigSchema `json:"admin_config"`
WorkerConfig WorkerConfigSchema `json:"worker_config"`
}
// AdminConfigSchema defines admin-side configuration
type AdminConfigSchema struct {
RebalanceInterval ConfigField `json:"rebalance_interval"`
MaxConcurrentJobs ConfigField `json:"max_concurrent_jobs"`
JobTimeout ConfigField `json:"job_timeout"`
HealthCheckInterval ConfigField `json:"health_check_interval"`
DiskUsageThreshold ConfigField `json:"disk_usage_threshold"`
AcceptableImbalancePercent ConfigField `json:"acceptable_imbalance_percent"`
}
// WorkerConfigSchema defines worker-side configuration
type WorkerConfigSchema struct {
MinVolumeSize ConfigField `json:"min_volume_size"`
MaxVolumeSize ConfigField `json:"max_volume_size"`
DataNodeCount ConfigField `json:"data_node_count"`
ReplicationFactor ConfigField `json:"replication_factor"`
PreferBalancedDistribution ConfigField `json:"prefer_balanced_distribution"`
}
// ConfigField describes a configuration field
type ConfigField struct {
Name string `json:"name"`
Description string `json:"description"`
Type string `json:"type"`
Required bool `json:"required"`
Default interface{} `json:"default,omitempty"`
Min interface{} `json:"min,omitempty"`
Max interface{} `json:"max,omitempty"`
Options []interface{} `json:"options,omitempty"`
Unit string `json:"unit,omitempty"`
}
// GetConfigurationSchema returns the schema for balance plugin configuration
func GetConfigurationSchema() *plugin_pb.PluginConfig {
schema := ConfigurationSchema{
AdminConfig: AdminConfigSchema{
RebalanceInterval: ConfigField{
Name: "rebalance_interval",
Description: "Time between rebalance scans",
Type: "duration",
Required: true,
Default: "2h",
Min: "30m",
Max: "12h",
Unit: "seconds",
},
MaxConcurrentJobs: ConfigField{
Name: "max_concurrent_jobs",
Description: "Maximum concurrent rebalance jobs",
Type: "integer",
Required: true,
Default: 2,
Min: 1,
Max: 5,
},
JobTimeout: ConfigField{
Name: "job_timeout",
Description: "Timeout for individual rebalance jobs",
Type: "duration",
Required: true,
Default: "6h",
Min: "1h",
Max: "24h",
Unit: "seconds",
},
HealthCheckInterval: ConfigField{
Name: "health_check_interval",
Description: "Health check interval",
Type: "duration",
Required: true,
Default: "30s",
Min: "5s",
Max: "5m",
Unit: "seconds",
},
DiskUsageThreshold: ConfigField{
Name: "disk_usage_threshold",
Description: "Disk usage threshold for triggering rebalance",
Type: "integer",
Required: true,
Default: 85,
Min: 50,
Max: 95,
Unit: "percent",
},
AcceptableImbalancePercent: ConfigField{
Name: "acceptable_imbalance_percent",
Description: "Acceptable imbalance percentage",
Type: "integer",
Required: true,
Default: 10,
Min: 1,
Max: 30,
Unit: "percent",
},
},
WorkerConfig: WorkerConfigSchema{
MinVolumeSize: ConfigField{
Name: "min_volume_size",
Description: "Minimum volume size to rebalance",
Type: "integer",
Required: true,
Default: 500,
Min: 100,
Unit: "MB",
},
MaxVolumeSize: ConfigField{
Name: "max_volume_size",
Description: "Maximum volume size to rebalance",
Type: "integer",
Required: true,
Default: 10000,
Max: 100000,
Unit: "MB",
},
DataNodeCount: ConfigField{
Name: "data_node_count",
Description: "Number of data nodes in cluster",
Type: "integer",
Required: true,
Default: 10,
Min: 1,
Max: 1000,
},
ReplicationFactor: ConfigField{
Name: "replication_factor",
Description: "Replication factor for volumes",
Type: "integer",
Required: true,
Default: 2,
Min: 1,
Max: 5,
},
PreferBalancedDistribution: ConfigField{
Name: "prefer_balanced_distribution",
Description: "Prefer balanced distribution",
Type: "boolean",
Required: true,
Default: true,
},
},
}
data, _ := json.MarshalIndent(schema, "", " ")
return &plugin_pb.PluginConfig{
PluginId: "balance-plugin",
Properties: map[string]string{
"schema": string(data),
"rebalance_interval": "2h",
"max_concurrent_jobs": "2",
"job_timeout": "6h",
"health_check_interval": "30s",
"disk_usage_threshold": "85",
"acceptable_imbalance_percent": "10",
"min_volume_size": "500",
"max_volume_size": "10000",
"data_node_count": "10",
"replication_factor": "2",
"prefer_balanced_distribution": "true",
},
}
}
// DefaultAdminConfig returns default admin configuration
func DefaultAdminConfig() map[string]string {
return map[string]string{
"rebalance_interval": "2h",
"max_concurrent_jobs": "2",
"job_timeout": "6h",
"health_check_interval": "30s",
"disk_usage_threshold": "85",
"acceptable_imbalance_percent": "10",
}
}
// DefaultWorkerConfig returns default worker configuration
func DefaultWorkerConfig() map[string]string {
return map[string]string{
"min_volume_size": "500",
"max_volume_size": "10000",
"data_node_count": "10",
"replication_factor": "2",
"prefer_balanced_distribution": "true",
}
}
-338
View File
@@ -1,338 +0,0 @@
package balance
import (
"context"
"flag"
"fmt"
"log"
"net"
"time"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// WorkerConfig holds worker-specific configuration
type WorkerConfig struct {
WorkerID string
AdminHost string
AdminPort int
PluginPort int
MinVolumeSize uint64
MaxVolumeSize uint64
DataNodeCount int
ReplicationFactor int
PreferBalancedDistribution bool
RebalanceInterval time.Duration
MaxConcurrentJobs int
HealthCheckInterval time.Duration
DiskUsageThreshold int
AcceptableImbalancePercent int
}
// Worker represents the balance plugin worker
type Worker struct {
config *WorkerConfig
pluginClient plugin_pb.PluginServiceClient
conn *grpc.ClientConn
detector *Detector
executor *Executor
activeJobs map[string]*plugin_pb.ExecuteJobRequest
done chan bool
isRunning bool
}
// NewWorker creates a new balance worker
func NewWorker(config *WorkerConfig) *Worker {
return &Worker{
config: config,
activeJobs: make(map[string]*plugin_pb.ExecuteJobRequest),
done: make(chan bool),
}
}
// Start initializes and starts the worker
func (w *Worker) Start(ctx context.Context) error {
log.Printf("Starting balance worker: %s", w.config.WorkerID)
// Connect to admin server
if err := w.connectToAdmin(ctx); err != nil {
return fmt.Errorf("failed to connect to admin: %v", err)
}
// Initialize detector
w.detector = NewDetector(DetectionOptions{
AcceptableImbalance: float32(w.config.AcceptableImbalancePercent),
DiskUsageThreshold: float32(w.config.DiskUsageThreshold),
MinVolumeSize: w.config.MinVolumeSize,
MaxVolumeSize: w.config.MaxVolumeSize,
PreferBalancedDist: w.config.PreferBalancedDistribution,
})
// Initialize executor
w.executor = NewExecutor(&ExecutorConfig{
MinVolumeSize: w.config.MinVolumeSize,
MaxVolumeSize: w.config.MaxVolumeSize,
TimeoutPerStep: 2 * time.Minute,
MaxRetries: 3,
})
// Register with admin
if err := w.registerPlugin(ctx); err != nil {
return fmt.Errorf("failed to register: %v", err)
}
w.isRunning = true
// Start background goroutines
go w.heartbeatLoop(ctx)
log.Printf("Balance worker started successfully")
return nil
}
// connectToAdmin establishes connection to admin server
func (w *Worker) connectToAdmin(ctx context.Context) error {
address := fmt.Sprintf("%s:%d", w.config.AdminHost, w.config.AdminPort)
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
conn, err := grpc.DialContext(dialCtx, address, grpc.WithInsecure())
if err != nil {
return fmt.Errorf("failed to dial: %v", err)
}
w.conn = conn
w.pluginClient = plugin_pb.NewPluginServiceClient(conn)
return nil
}
// registerPlugin registers the plugin with the admin server
func (w *Worker) registerPlugin(ctx context.Context) error {
schema := GetConfigurationSchema()
req := &plugin_pb.PluginConnectRequest{
PluginId: w.config.WorkerID,
PluginName: "balance-plugin",
Version: "1.0.0",
Capabilities: []string{"detect", "execute", "report_health"},
MaxConcurrentJobs: int32(w.config.MaxConcurrentJobs),
SupportsStreaming: true,
Port: int32(w.config.PluginPort),
}
// Add capabilities detail
req.CapabilitiesDetail = &plugin_pb.PluginCapabilities{
Detection: []*plugin_pb.DetectionCapability{
{
Type: "rebalance_candidates",
Description: "Detect nodes that need rebalancing",
MinIntervalSeconds: int32(w.config.RebalanceInterval.Seconds()),
RequiresFullScan: true,
},
},
Maintenance: []*plugin_pb.MaintenanceCapability{
{
Type: "rebalance_data",
Description: "Rebalance data across nodes",
RequiredDetectionTypes: []string{"rebalance_candidates"},
EstimatedDurationSeconds: 3600,
},
},
}
// Add schema to metadata
if schema != nil {
if req.Metadata == nil {
req.Metadata = make(map[string]string)
}
for k, v := range schema.Properties {
req.Metadata[k] = v
}
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
resp, err := w.pluginClient.Connect(ctx, req)
if err != nil {
return fmt.Errorf("connect RPC failed: %v", err)
}
if !resp.Success {
return fmt.Errorf("connect failed: %s", resp.Message)
}
log.Printf("Plugin registered with master: %s", resp.MasterId)
return nil
}
// heartbeatLoop sends periodic health reports
func (w *Worker) heartbeatLoop(ctx context.Context) {
ticker := time.NewTicker(w.config.HealthCheckInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-w.done:
return
case <-ticker.C:
w.sendHealthReport(ctx)
}
}
}
// sendHealthReport sends a health report to the admin
func (w *Worker) sendHealthReport(ctx context.Context) {
report := &plugin_pb.HealthReport{
PluginId: w.config.WorkerID,
TimestampMs: time.Now().UnixMilli(),
Status: plugin_pb.HealthStatus_HEALTH_STATUS_HEALTHY,
ActiveJobs: int32(len(w.activeJobs)),
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
_, err := w.pluginClient.ReportHealth(ctx, report)
if err != nil {
log.Printf("Failed to send health report: %v", err)
}
}
// ExecuteDetection performs detection for rebalance opportunities
func (w *Worker) ExecuteDetection(ctx context.Context, nodeMetrics map[string]*NodeMetric) ([]*RebalanceCandidate, error) {
return w.detector.DetectJobs(nodeMetrics)
}
// ExecuteJob executes a rebalance job
func (w *Worker) ExecuteJob(ctx context.Context, jobID string, payload *plugin_pb.JobPayload, source, dest string) error {
req := &plugin_pb.ExecuteJobRequest{
JobId: jobID,
JobType: "rebalance_data",
Payload: payload,
RetryCount: 0,
}
w.activeJobs[jobID] = req
defer delete(w.activeJobs, jobID)
// Execute the job
result, err := w.executor.ExecuteJob(req, source, dest)
if err != nil {
log.Printf("Job execution failed: %v", err)
return err
}
if result.Success {
log.Printf("Job %s completed successfully", jobID)
return w.submitResult(ctx, jobID, result)
}
log.Printf("Job %s failed: %s", jobID, result.ErrorMessage)
return fmt.Errorf("%s", result.ErrorMessage)
}
// submitResult submits job results to admin
func (w *Worker) submitResult(ctx context.Context, jobID string, result *BalanceExecutionResult) error {
jobResult := &plugin_pb.JobResult{
Success: result.Success,
Metadata: result.Metadata,
}
req := &plugin_pb.JobResultRequest{
JobId: jobID,
JobType: "rebalance_data",
Status: plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED,
Message: "Rebalancing completed successfully",
Result: jobResult,
RetryCountUsed: 0,
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
_, err := w.pluginClient.SubmitResult(ctx, req)
return err
}
// Stop gracefully stops the worker
func (w *Worker) Stop(ctx context.Context) error {
log.Printf("Stopping balance worker")
w.isRunning = false
close(w.done)
if w.conn != nil {
return w.conn.Close()
}
return nil
}
// GetStatus returns the current worker status
func (w *Worker) GetStatus() map[string]interface{} {
return map[string]interface{}{
"worker_id": w.config.WorkerID,
"is_running": w.isRunning,
"active_jobs": len(w.activeJobs),
"admin_connected": w.conn != nil,
}
}
// ParseFlags parses command line flags for balance worker
func ParseFlags() *WorkerConfig {
config := &WorkerConfig{
WorkerID: "balance-worker-1",
AdminHost: "localhost",
AdminPort: 50051,
PluginPort: 50054,
MinVolumeSize: 500,
MaxVolumeSize: 10000,
DataNodeCount: 10,
ReplicationFactor: 2,
PreferBalancedDistribution: true,
RebalanceInterval: 2 * time.Hour,
MaxConcurrentJobs: 2,
HealthCheckInterval: 30 * time.Second,
DiskUsageThreshold: 85,
AcceptableImbalancePercent: 10,
}
flag.StringVar(&config.WorkerID, "worker-id", config.WorkerID, "Worker ID")
flag.StringVar(&config.AdminHost, "admin-host", config.AdminHost, "Admin server host")
flag.IntVar(&config.AdminPort, "admin-port", config.AdminPort, "Admin server port")
flag.IntVar(&config.PluginPort, "plugin-port", config.PluginPort, "Plugin server port")
flag.Uint64Var(&config.MinVolumeSize, "min-volume-size", config.MinVolumeSize, "Minimum volume size in MB")
flag.Uint64Var(&config.MaxVolumeSize, "max-volume-size", config.MaxVolumeSize, "Maximum volume size in MB")
flag.IntVar(&config.DataNodeCount, "data-node-count", config.DataNodeCount, "Data node count")
flag.IntVar(&config.ReplicationFactor, "replication-factor", config.ReplicationFactor, "Replication factor")
flag.BoolVar(&config.PreferBalancedDistribution, "prefer-balanced", config.PreferBalancedDistribution, "Prefer balanced distribution")
flag.DurationVar(&config.RebalanceInterval, "rebalance-interval", config.RebalanceInterval, "Rebalance interval")
flag.IntVar(&config.MaxConcurrentJobs, "max-concurrent-jobs", config.MaxConcurrentJobs, "Max concurrent jobs")
flag.DurationVar(&config.HealthCheckInterval, "health-check-interval", config.HealthCheckInterval, "Health check interval")
flag.IntVar(&config.DiskUsageThreshold, "disk-usage-threshold", config.DiskUsageThreshold, "Disk usage threshold percent")
flag.IntVar(&config.AcceptableImbalancePercent, "acceptable-imbalance", config.AcceptableImbalancePercent, "Acceptable imbalance percent")
flag.Parse()
return config
}
// ListenAndServe starts the gRPC server for the worker
func (w *Worker) ListenAndServe(port int) error {
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
return fmt.Errorf("failed to listen on port %d: %v", port, err)
}
server := grpc.NewServer()
log.Printf("Worker listening on port %d", port)
return server.Serve(listener)
}
@@ -1,227 +0,0 @@
package erasure_coding
import (
"fmt"
)
// CandidateVolume represents a volume eligible for EC
type CandidateVolume struct {
VolumeID uint32
DataNodeID string
Size uint64
FreeSpace uint64
ReplicaCount int
RackID string
DataCenterID string
FileCount int64
LastModified int64
CanEncode bool
Reason string
}
// DetectionOptions contains options for detection
type DetectionOptions struct {
MinVolumeSize uint64
MaxVolumeSize uint64
RackAwareness bool
DataCenterAwareness bool
PreferredNodes []string
ExcludeNodes []string
}
// Detector scans for EC candidates
type Detector struct {
config DetectionOptions
}
// NewDetector creates a new EC detector
func NewDetector(opts DetectionOptions) *Detector {
return &Detector{
config: opts,
}
}
// DetectJobs scans volumes for EC candidates
func (d *Detector) DetectJobs(volumeMetrics map[uint32]*VolumeMetric) ([]*CandidateVolume, error) {
candidates := make([]*CandidateVolume, 0)
for volumeID, metric := range volumeMetrics {
candidate, shouldInclude := d.evaluateVolume(volumeID, metric)
if shouldInclude {
candidates = append(candidates, candidate)
}
}
return candidates, nil
}
// evaluateVolume checks if a volume should be encoded
func (d *Detector) evaluateVolume(volumeID uint32, metric *VolumeMetric) (*CandidateVolume, bool) {
candidate := &CandidateVolume{
VolumeID: volumeID,
DataNodeID: metric.DataNodeID,
Size: metric.Size,
FreeSpace: metric.FreeSpace,
ReplicaCount: metric.ReplicaCount,
RackID: metric.RackID,
DataCenterID: metric.DataCenterID,
FileCount: metric.FileCount,
LastModified: metric.LastModified,
}
// Check size constraints
if metric.Size < d.config.MinVolumeSize {
candidate.CanEncode = false
candidate.Reason = fmt.Sprintf("volume too small: %d < %d", metric.Size, d.config.MinVolumeSize)
return candidate, false
}
if metric.Size > d.config.MaxVolumeSize {
candidate.CanEncode = false
candidate.Reason = fmt.Sprintf("volume too large: %d > %d", metric.Size, d.config.MaxVolumeSize)
return candidate, false
}
// Check if already encoded
if metric.IsEncoded {
candidate.CanEncode = false
candidate.Reason = "volume already encoded"
return candidate, false
}
// Check replica count for optimization potential
if metric.ReplicaCount <= 1 {
candidate.CanEncode = false
candidate.Reason = "insufficient replication for encoding"
return candidate, false
}
// Check node exclusion
if d.isNodeExcluded(metric.DataNodeID) {
candidate.CanEncode = false
candidate.Reason = "node is in exclusion list"
return candidate, false
}
// Check node preference
if len(d.config.PreferredNodes) > 0 && !d.isPreferredNode(metric.DataNodeID) {
candidate.CanEncode = false
candidate.Reason = "node not in preferred list"
return candidate, false
}
// Check rack awareness if enabled
if d.config.RackAwareness && metric.RackID == "" {
candidate.CanEncode = false
candidate.Reason = "rack awareness enabled but no rack information"
return candidate, false
}
// Check data center awareness if enabled
if d.config.DataCenterAwareness && metric.DataCenterID == "" {
candidate.CanEncode = false
candidate.Reason = "data center awareness enabled but no data center information"
return candidate, false
}
// Check if volume has aged enough (at least 1 hour old)
if metric.LastModified == 0 {
candidate.CanEncode = false
candidate.Reason = "volume too new, needs aging"
return candidate, false
}
// Volume is a candidate
candidate.CanEncode = true
candidate.Reason = "eligible for erasure coding"
return candidate, true
}
// isNodeExcluded checks if a node is in the exclusion list
func (d *Detector) isNodeExcluded(nodeID string) bool {
for _, excluded := range d.config.ExcludeNodes {
if excluded == nodeID {
return true
}
}
return false
}
// isPreferredNode checks if a node is in the preferred list
func (d *Detector) isPreferredNode(nodeID string) bool {
for _, preferred := range d.config.PreferredNodes {
if preferred == nodeID {
return true
}
}
return false
}
// VolumeMetric contains volume statistics
type VolumeMetric struct {
VolumeID uint32
DataNodeID string
Size uint64
FreeSpace uint64
ReplicaCount int
RackID string
DataCenterID string
FileCount int64
LastModified int64
IsEncoded bool
CanResize bool
Collection string
CompactionSize int64
}
// FilterByCriteria filters volumes by specific criteria
func FilterByCriteria(candidates []*CandidateVolume, criteria map[string]string) []*CandidateVolume {
filtered := make([]*CandidateVolume, 0)
for _, candidate := range candidates {
if !candidate.CanEncode {
continue
}
// Apply collection filter if specified
if collection, ok := criteria["collection"]; ok && collection != "" {
// Would need collection info in candidate; skipping for now
continue
}
// Apply rack filter if specified
if rack, ok := criteria["rack"]; ok && rack != "" && candidate.RackID != rack {
continue
}
// Apply data center filter if specified
if dc, ok := criteria["datacenter"]; ok && dc != "" && candidate.DataCenterID != dc {
continue
}
filtered = append(filtered, candidate)
}
return filtered
}
// SortByPriority sorts candidates by encoding priority
func SortByPriority(candidates []*CandidateVolume) {
// In a real implementation, this would use a sorting algorithm
// For now, candidates are already in order
}
// GroupByRack groups candidates by rack for distributed encoding
func GroupByRack(candidates []*CandidateVolume) map[string][]*CandidateVolume {
grouped := make(map[string][]*CandidateVolume)
for _, candidate := range candidates {
rackID := candidate.RackID
if rackID == "" {
rackID = "default"
}
grouped[rackID] = append(grouped[rackID], candidate)
}
return grouped
}
@@ -1,296 +0,0 @@
package erasure_coding
import (
"context"
"testing"
"time"
plugin_testing "github.com/seaweedfs/seaweedfs/weed/admin/plugin/testing"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// TestDetectionWithSingleVolume tests detection of a single volume
func TestDetectionWithSingleVolume(t *testing.T) {
harness := plugin_testing.NewTestHarness("TestDetectionWithSingleVolume")
defer harness.Cleanup()
// Create and register a mock plugin
plugin := plugin_testing.NewMockPlugin("ec-worker-1", "EC Plugin", "1.0.0")
plugin.AddDetectionCapability("ec_candidates", "Detect EC candidates", 3600, true)
if err := harness.RegisterPlugin(plugin); err != nil {
t.Fatalf("Failed to register plugin: %v", err)
}
if !harness.VerifyRegistration("ec-worker-1") {
t.Error("Plugin registration not verified")
}
if harness.GetRegistrationCount() != 1 {
t.Errorf("Expected 1 registration, got %d", harness.GetRegistrationCount())
}
}
// TestDetectionWithMultipleVolumes tests detection of multiple volumes
func TestDetectionWithMultipleVolumes(t *testing.T) {
harness := plugin_testing.NewTestHarness("TestDetectionWithMultipleVolumes")
defer harness.Cleanup()
plugin := plugin_testing.NewMockPlugin("ec-worker-2", "EC Plugin", "1.0.0")
plugin.AddDetectionCapability("ec_candidates", "Detect EC candidates", 3600, true)
// Add multiple detection results
plugin.AddDetectionResult("vol-1", "ec_candidates", "info", "Volume 1 candidate", nil)
plugin.AddDetectionResult("vol-2", "ec_candidates", "info", "Volume 2 candidate", nil)
plugin.AddDetectionResult("vol-3", "ec_candidates", "info", "Volume 3 candidate", nil)
if err := harness.RegisterPlugin(plugin); err != nil {
t.Fatalf("Failed to register plugin: %v", err)
}
// Verify capabilities
if !harness.VerifyPluginCapability("ec-worker-2", "ec_candidates") {
t.Error("EC candidates capability not found")
}
}
// TestJobDispatch tests job dispatch to EC plugin
func TestJobDispatch(t *testing.T) {
harness := plugin_testing.NewTestHarness("TestJobDispatch")
defer harness.Cleanup()
plugin := plugin_testing.NewMockPlugin("ec-worker-3", "EC Plugin", "1.0.0")
plugin.AddDetectionCapability("ec_candidates", "Detect EC candidates", 3600, true)
if err := harness.RegisterPlugin(plugin); err != nil {
t.Fatalf("Failed to register plugin: %v", err)
}
// Dispatch a job
payload := &plugin_pb.JobPayload{
DetectionType: "encode_volume",
TargetDatasource: "volume-123",
Data: []byte{1, 2, 3, 4},
Parameters: map[string]string{"stripe_size": "10"},
}
jobID, err := harness.DispatchJob("ec-worker-3", "encode_volume", payload)
if err != nil {
t.Fatalf("Failed to dispatch job: %v", err)
}
if jobID == "" {
t.Error("No job ID returned")
}
// Verify job was dispatched
if harness.GetJobCount() != 1 {
t.Errorf("Expected 1 job, got %d", harness.GetJobCount())
}
// Verify job completed
if !harness.VerifyJobCompleted(jobID) {
t.Errorf("Job %s did not complete", jobID)
}
}
// TestExecutionPipeline tests the full EC execution pipeline
func TestExecutionPipeline(t *testing.T) {
executor := NewExecutor(&ExecutorConfig{
StripeSize: 10,
EncodeCopies: 1,
TimeoutPerStep: 1 * time.Second,
MaxRetries: 3,
})
// Create a mock job
job := &plugin_pb.ExecuteJobRequest{
JobId: "job-123",
JobType: "encode_volume",
Payload: &plugin_pb.JobPayload{Data: []byte{1, 2, 3, 4}},
RetryCount: 0,
}
result, err := executor.ExecuteJob(job)
if err != nil {
t.Fatalf("Execution failed: %v", err)
}
if !result.Success {
t.Errorf("Execution was not successful: %s", result.ErrorMessage)
}
if len(result.Steps) != 6 {
t.Errorf("Expected 6 steps, got %d", len(result.Steps))
}
// Verify pipeline steps
expectedSteps := []string{"marking", "copying", "generating", "distributing", "mounting", "cleaning"}
for i, expected := range expectedSteps {
if i >= len(result.Steps) {
t.Errorf("Missing step: %s", expected)
break
}
if result.Steps[i].Name != expected {
t.Errorf("Step %d: expected %s, got %s", i, expected, result.Steps[i].Name)
}
}
}
// TestErrorHandling tests error handling in execution
func TestErrorHandling(t *testing.T) {
harness := plugin_testing.NewTestHarness("TestErrorHandling")
defer harness.Cleanup()
plugin := plugin_testing.NewMockPlugin("ec-worker-4", "EC Plugin", "1.0.0")
plugin.AddDetectionCapability("ec_candidates", "Detect EC candidates", 3600, true)
if err := harness.RegisterPlugin(plugin); err != nil {
t.Fatalf("Failed to register plugin: %v", err)
}
// Verify plugin is registered
if !harness.VerifyRegistration("ec-worker-4") {
t.Error("Plugin registration not verified")
}
}
// TestDetectorFiltering tests volume filtering in detector
func TestDetectorFiltering(t *testing.T) {
detector := NewDetector(DetectionOptions{
MinVolumeSize: 1000,
MaxVolumeSize: 10000,
RackAwareness: true,
})
// Create test volumes
volumes := map[uint32]*VolumeMetric{
1: {
VolumeID: 1,
Size: 500, // Too small
FreeSpace: 100,
ReplicaCount: 2,
LastModified: 1,
},
2: {
VolumeID: 2,
Size: 5000, // Good
FreeSpace: 1000,
ReplicaCount: 2,
RackID: "rack-1",
LastModified: 1,
},
3: {
VolumeID: 3,
Size: 20000, // Too large
FreeSpace: 5000,
ReplicaCount: 2,
LastModified: 1,
},
4: {
VolumeID: 4,
Size: 3000, // Good but already encoded
IsEncoded: true,
FreeSpace: 500,
ReplicaCount: 2,
LastModified: 1,
},
}
candidates, err := detector.DetectJobs(volumes)
if err != nil {
t.Fatalf("Detection failed: %v", err)
}
if len(candidates) != 1 {
t.Errorf("Expected 1 candidate, got %d", len(candidates))
for _, c := range candidates {
t.Logf("Candidate: %d - %s", c.VolumeID, c.Reason)
}
return
}
if len(candidates) > 0 && candidates[0].VolumeID != 2 {
t.Errorf("Expected volume 2, got %d", candidates[0].VolumeID)
}
}
// TestHealthReporting tests health report submission
func TestHealthReporting(t *testing.T) {
harness := plugin_testing.NewTestHarness("TestHealthReporting")
defer harness.Cleanup()
plugin := plugin_testing.NewMockPlugin("ec-worker-5", "EC Plugin", "1.0.0")
if err := harness.RegisterPlugin(plugin); err != nil {
t.Fatalf("Failed to register plugin: %v", err)
}
adminService := harness.GetAdminService()
// Send health report
report := &plugin_pb.HealthReport{
PluginId: "ec-worker-5",
TimestampMs: time.Now().UnixMilli(),
Status: plugin_pb.HealthStatus_HEALTH_STATUS_HEALTHY,
ActiveJobs: 3,
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
resp, err := adminService.ReportHealth(ctx, report)
if err != nil {
t.Fatalf("Failed to report health: %v", err)
}
if !resp.Acknowledged {
t.Error("Health report not acknowledged")
}
if adminService.GetHeartbeatCount() != 1 {
t.Errorf("Expected 1 heartbeat, got %d", adminService.GetHeartbeatCount())
}
}
// TestConcurrentJobExecution tests multiple concurrent jobs
func TestConcurrentJobExecution(t *testing.T) {
harness := plugin_testing.NewTestHarness("TestConcurrentJobExecution")
defer harness.Cleanup()
plugin := plugin_testing.NewMockPlugin("ec-worker-6", "EC Plugin", "1.0.0")
if err := harness.RegisterPlugin(plugin); err != nil {
t.Fatalf("Failed to register plugin: %v", err)
}
// Dispatch multiple jobs
jobIDs := make([]string, 0)
for i := 0; i < 5; i++ {
payload := &plugin_pb.JobPayload{
DetectionType: "encode_volume",
Data: []byte{byte(i)},
}
jobID, err := harness.DispatchJob("ec-worker-6", "encode_volume", payload)
if err != nil {
t.Fatalf("Failed to dispatch job %d: %v", i, err)
}
jobIDs = append(jobIDs, jobID)
}
// Verify all jobs
if harness.GetJobCount() != 5 {
t.Errorf("Expected 5 jobs, got %d", harness.GetJobCount())
}
completedCount := 0
for _, jobID := range jobIDs {
if harness.VerifyJobCompleted(jobID) {
completedCount++
}
}
if completedCount != 5 {
t.Errorf("Expected 5 completed jobs, got %d", completedCount)
}
}
@@ -1,304 +0,0 @@
package erasure_coding
import (
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// ExecutionStatus tracks job execution status
type ExecutionStatus string
const (
StatusMarking ExecutionStatus = "marking"
StatusCopying ExecutionStatus = "copying"
StatusGenerating ExecutionStatus = "generating"
StatusDistributing ExecutionStatus = "distributing"
StatusMounting ExecutionStatus = "mounting"
StatusCleaning ExecutionStatus = "cleaning"
StatusCompleted ExecutionStatus = "completed"
StatusFailed ExecutionStatus = "failed"
)
// ExecutionStep represents a step in the EC encoding pipeline
type ExecutionStep struct {
Name string
Status ExecutionStatus
StartTime *time.Time
EndTime *time.Time
Progress float32
ErrorMsg string
}
// Executor handles EC encoding execution
type Executor struct {
config *ExecutorConfig
}
// ExecutorConfig contains executor configuration
type ExecutorConfig struct {
StripeSize int
EncodeCopies int
RackAwareness bool
DataCenterAwareness bool
TimeoutPerStep time.Duration
MaxRetries int
}
// NewExecutor creates a new EC executor
func NewExecutor(config *ExecutorConfig) *Executor {
if config == nil {
config = &ExecutorConfig{
StripeSize: 10,
EncodeCopies: 1,
TimeoutPerStep: 5 * time.Minute,
MaxRetries: 3,
}
}
return &Executor{config: config}
}
// ExecutionResult contains the result of encoding
type ExecutionResult struct {
VolumeID uint32
Success bool
StartTime time.Time
EndTime time.Time
TotalDuration time.Duration
BytesProcessed uint64
StripeCount int
Metadata map[string]string
Steps []*ExecutionStep
ErrorMessage string
}
// ExecuteJob executes the EC encoding for a volume
func (e *Executor) ExecuteJob(job *plugin_pb.ExecuteJobRequest) (*ExecutionResult, error) {
result := &ExecutionResult{
Success: false,
StartTime: time.Now(),
Metadata: make(map[string]string),
Steps: make([]*ExecutionStep, 0),
}
// Extract volume ID from payload
volumeID := extractVolumeID(job.Payload)
result.VolumeID = volumeID
// Step 1: Mark volume as being encoded
if err := e.markVolume(result); err != nil {
result.ErrorMessage = fmt.Sprintf("mark failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 2: Copy volume data
if err := e.copyVolumeData(result); err != nil {
result.ErrorMessage = fmt.Sprintf("copy failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 3: Generate parity shards
if err := e.generateParityShards(result); err != nil {
result.ErrorMessage = fmt.Sprintf("parity generation failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 4: Distribute shards across nodes
if err := e.distributeShards(result); err != nil {
result.ErrorMessage = fmt.Sprintf("distribution failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 5: Mount new EC volume
if err := e.mountECVolume(result); err != nil {
result.ErrorMessage = fmt.Sprintf("mount failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 6: Delete original replicas
if err := e.deleteOriginalReplicas(result); err != nil {
result.ErrorMessage = fmt.Sprintf("cleanup failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
result.Success = true
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, nil
}
// markVolume marks the volume as being encoded
func (e *Executor) markVolume(result *ExecutionResult) error {
step := &ExecutionStep{
Name: "marking",
Status: StatusMarking,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
// Simulate marking operation
time.Sleep(50 * time.Millisecond)
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// copyVolumeData copies volume data to temporary location
func (e *Executor) copyVolumeData(result *ExecutionResult) error {
step := &ExecutionStep{
Name: "copying",
Status: StatusCopying,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
// Simulate data copying with progress
for i := 0; i < 10; i++ {
time.Sleep(10 * time.Millisecond)
step.Progress = float32((i + 1) * 10)
}
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
result.BytesProcessed += 1000000 // Simulate processing
return nil
}
// generateParityShards generates parity shards from original data
func (e *Executor) generateParityShards(result *ExecutionResult) error {
step := &ExecutionStep{
Name: "generating",
Status: StatusGenerating,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
// Simulate parity generation
time.Sleep(100 * time.Millisecond)
result.StripeCount = int(result.BytesProcessed / uint64(e.config.StripeSize*1024*1024))
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// distributeShards distributes shards across data nodes
func (e *Executor) distributeShards(result *ExecutionResult) error {
step := &ExecutionStep{
Name: "distributing",
Status: StatusDistributing,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
// Simulate shard distribution
for i := 0; i < 5; i++ {
time.Sleep(20 * time.Millisecond)
step.Progress = float32((i + 1) * 20)
}
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// mountECVolume mounts the new EC volume
func (e *Executor) mountECVolume(result *ExecutionResult) error {
step := &ExecutionStep{
Name: "mounting",
Status: StatusMounting,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
// Simulate mounting
time.Sleep(50 * time.Millisecond)
result.Metadata["ec_volume_id"] = fmt.Sprintf("%d.ec", result.VolumeID)
result.Metadata["stripe_size"] = fmt.Sprintf("%d MB", e.config.StripeSize)
result.Metadata["encode_copies"] = fmt.Sprintf("%d", e.config.EncodeCopies)
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// deleteOriginalReplicas deletes the original replica volumes
func (e *Executor) deleteOriginalReplicas(result *ExecutionResult) error {
step := &ExecutionStep{
Name: "cleaning",
Status: StatusCleaning,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
// Simulate cleanup
time.Sleep(50 * time.Millisecond)
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// extractVolumeID extracts the volume ID from job payload
func extractVolumeID(payload *plugin_pb.JobPayload) uint32 {
if payload == nil || len(payload.Data) < 4 {
return 0
}
// Simple extraction: first 4 bytes as little-endian uint32
return uint32(payload.Data[0]) |
(uint32(payload.Data[1]) << 8) |
(uint32(payload.Data[2]) << 16) |
(uint32(payload.Data[3]) << 24)
}
// ValidateExecutionResult validates the result of execution
func ValidateExecutionResult(result *ExecutionResult) bool {
if !result.Success {
return false
}
if result.EndTime.Before(result.StartTime) {
return false
}
if len(result.Steps) != 6 {
return false
}
return true
}
@@ -1,205 +0,0 @@
package erasure_coding
import (
"encoding/json"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// ConfigurationSchema defines the schema for EC plugin configuration
type ConfigurationSchema struct {
AdminConfig AdminConfigSchema `json:"admin_config"`
WorkerConfig WorkerConfigSchema `json:"worker_config"`
}
// AdminConfigSchema defines admin-side configuration
type AdminConfigSchema struct {
DetectionInterval ConfigField `json:"detection_interval"`
MaxConcurrentJobs ConfigField `json:"max_concurrent_jobs"`
JobTimeout ConfigField `json:"job_timeout"`
HealthCheckInterval ConfigField `json:"health_check_interval"`
RetryPolicy ConfigField `json:"retry_policy"`
}
// WorkerConfigSchema defines worker-side configuration
type WorkerConfigSchema struct {
StripeSize ConfigField `json:"stripe_size"`
EncodeCopies ConfigField `json:"encode_copies"`
RackAwareness ConfigField `json:"rack_awareness"`
DataCenterAwareness ConfigField `json:"datacenter_awareness"`
MinVolumeSize ConfigField `json:"min_volume_size"`
MaxVolumeSize ConfigField `json:"max_volume_size"`
PreferredDataNodes ConfigField `json:"preferred_data_nodes"`
}
// ConfigField describes a configuration field
type ConfigField struct {
Name string `json:"name"`
Description string `json:"description"`
Type string `json:"type"`
Required bool `json:"required"`
Default interface{} `json:"default,omitempty"`
Min interface{} `json:"min,omitempty"`
Max interface{} `json:"max,omitempty"`
Options []interface{} `json:"options,omitempty"`
Unit string `json:"unit,omitempty"`
}
// GetConfigurationSchema returns the schema for EC plugin configuration
func GetConfigurationSchema() *plugin_pb.PluginConfig {
schema := ConfigurationSchema{
AdminConfig: AdminConfigSchema{
DetectionInterval: ConfigField{
Name: "detection_interval",
Description: "Time between EC detection scans",
Type: "duration",
Required: true,
Default: "1h",
Min: "5m",
Max: "24h",
Unit: "seconds",
},
MaxConcurrentJobs: ConfigField{
Name: "max_concurrent_jobs",
Description: "Maximum concurrent EC encoding jobs",
Type: "integer",
Required: true,
Default: 5,
Min: 1,
Max: 20,
},
JobTimeout: ConfigField{
Name: "job_timeout",
Description: "Timeout for individual EC jobs",
Type: "duration",
Required: true,
Default: "12h",
Min: "1h",
Max: "48h",
Unit: "seconds",
},
HealthCheckInterval: ConfigField{
Name: "health_check_interval",
Description: "Health check interval",
Type: "duration",
Required: true,
Default: "30s",
Min: "5s",
Max: "5m",
Unit: "seconds",
},
RetryPolicy: ConfigField{
Name: "retry_policy",
Description: "Retry policy for failed jobs",
Type: "string",
Required: true,
Default: "exponential",
Options: []interface{}{"linear", "exponential", "none"},
},
},
WorkerConfig: WorkerConfigSchema{
StripeSize: ConfigField{
Name: "stripe_size",
Description: "Size of each stripe in MB",
Type: "integer",
Required: true,
Default: 10,
Min: 1,
Max: 100,
Unit: "MB",
},
EncodeCopies: ConfigField{
Name: "encode_copies",
Description: "Number of copies to keep after encoding",
Type: "integer",
Required: true,
Default: 1,
Min: 1,
Max: 3,
},
RackAwareness: ConfigField{
Name: "rack_awareness",
Description: "Enable rack-aware stripe distribution",
Type: "boolean",
Required: true,
Default: true,
},
DataCenterAwareness: ConfigField{
Name: "datacenter_awareness",
Description: "Enable data center aware stripe distribution",
Type: "boolean",
Required: true,
Default: false,
},
MinVolumeSize: ConfigField{
Name: "min_volume_size",
Description: "Minimum volume size to consider for EC",
Type: "integer",
Required: true,
Default: 1000,
Min: 100,
Unit: "MB",
},
MaxVolumeSize: ConfigField{
Name: "max_volume_size",
Description: "Maximum volume size to consider for EC",
Type: "integer",
Required: true,
Default: 10000,
Max: 100000,
Unit: "MB",
},
PreferredDataNodes: ConfigField{
Name: "preferred_data_nodes",
Description: "Comma-separated list of preferred data nodes",
Type: "string",
Required: false,
},
},
}
data, _ := json.MarshalIndent(schema, "", " ")
return &plugin_pb.PluginConfig{
PluginId: "erasure-coding-plugin",
Properties: map[string]string{
"schema": string(data),
"detection_interval": "1h",
"max_concurrent_jobs": "5",
"job_timeout": "12h",
"health_check_interval": "30s",
"retry_policy": "exponential",
"stripe_size": "10",
"encode_copies": "1",
"rack_awareness": "true",
"datacenter_awareness": "false",
"min_volume_size": "1000",
"max_volume_size": "10000",
"preferred_data_nodes": "",
},
}
}
// DefaultAdminConfig returns default admin configuration
func DefaultAdminConfig() map[string]string {
return map[string]string{
"detection_interval": "1h",
"max_concurrent_jobs": "5",
"job_timeout": "12h",
"health_check_interval": "30s",
"retry_policy": "exponential",
}
}
// DefaultWorkerConfig returns default worker configuration
func DefaultWorkerConfig() map[string]string {
return map[string]string{
"stripe_size": "10",
"encode_copies": "1",
"rack_awareness": "true",
"datacenter_awareness": "false",
"min_volume_size": "1000",
"max_volume_size": "10000",
"preferred_data_nodes": "",
}
}
@@ -1,341 +0,0 @@
package erasure_coding
import (
"context"
"flag"
"fmt"
"log"
"net"
"time"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// WorkerConfig holds worker-specific configuration
type WorkerConfig struct {
WorkerID string
AdminHost string
AdminPort int
PluginPort int
StripeSize int
EncodeCopies int
RackAwareness bool
DataCenterAwareness bool
MinVolumeSize uint64
MaxVolumeSize uint64
DetectionInterval time.Duration
MaxConcurrentJobs int
HealthCheckInterval time.Duration
RetryPolicy string
}
// Worker represents the EC plugin worker
type Worker struct {
config *WorkerConfig
pluginClient plugin_pb.PluginServiceClient
conn *grpc.ClientConn
detector *Detector
executor *Executor
activeJobs map[string]*plugin_pb.ExecuteJobRequest
done chan bool
isRunning bool
}
// NewWorker creates a new EC worker
func NewWorker(config *WorkerConfig) *Worker {
return &Worker{
config: config,
activeJobs: make(map[string]*plugin_pb.ExecuteJobRequest),
done: make(chan bool),
}
}
// Start initializes and starts the worker
func (w *Worker) Start(ctx context.Context) error {
log.Printf("Starting EC worker: %s", w.config.WorkerID)
// Connect to admin server
if err := w.connectToAdmin(ctx); err != nil {
return fmt.Errorf("failed to connect to admin: %v", err)
}
// Initialize detector
w.detector = NewDetector(DetectionOptions{
MinVolumeSize: w.config.MinVolumeSize,
MaxVolumeSize: w.config.MaxVolumeSize,
RackAwareness: w.config.RackAwareness,
DataCenterAwareness: w.config.DataCenterAwareness,
})
// Initialize executor
w.executor = NewExecutor(&ExecutorConfig{
StripeSize: w.config.StripeSize,
EncodeCopies: w.config.EncodeCopies,
RackAwareness: w.config.RackAwareness,
DataCenterAwareness: w.config.DataCenterAwareness,
TimeoutPerStep: 5 * time.Minute,
MaxRetries: 3,
})
// Register with admin
if err := w.registerPlugin(ctx); err != nil {
return fmt.Errorf("failed to register: %v", err)
}
w.isRunning = true
// Start background goroutines
go w.heartbeatLoop(ctx)
log.Printf("EC worker started successfully")
return nil
}
// connectToAdmin establishes connection to admin server
func (w *Worker) connectToAdmin(ctx context.Context) error {
address := fmt.Sprintf("%s:%d", w.config.AdminHost, w.config.AdminPort)
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
conn, err := grpc.DialContext(dialCtx, address, grpc.WithInsecure())
if err != nil {
return fmt.Errorf("failed to dial: %v", err)
}
w.conn = conn
w.pluginClient = plugin_pb.NewPluginServiceClient(conn)
return nil
}
// registerPlugin registers the plugin with the admin server
func (w *Worker) registerPlugin(ctx context.Context) error {
schema := GetConfigurationSchema()
req := &plugin_pb.PluginConnectRequest{
PluginId: w.config.WorkerID,
PluginName: "erasure-coding-plugin",
Version: "1.0.0",
Capabilities: []string{"detect", "execute", "report_health"},
MaxConcurrentJobs: int32(w.config.MaxConcurrentJobs),
SupportsStreaming: true,
Port: int32(w.config.PluginPort),
}
// Add capabilities detail
req.CapabilitiesDetail = &plugin_pb.PluginCapabilities{
Detection: []*plugin_pb.DetectionCapability{
{
Type: "ec_candidates",
Description: "Detect volumes eligible for erasure coding",
MinIntervalSeconds: int32(w.config.DetectionInterval.Seconds()),
RequiresFullScan: true,
},
},
Maintenance: []*plugin_pb.MaintenanceCapability{
{
Type: "encode_volume",
Description: "Encode a volume with erasure coding",
RequiredDetectionTypes: []string{"ec_candidates"},
EstimatedDurationSeconds: 3600,
},
},
}
// Add schema to metadata
if schema != nil {
if req.Metadata == nil {
req.Metadata = make(map[string]string)
}
for k, v := range schema.Properties {
req.Metadata[k] = v
}
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
resp, err := w.pluginClient.Connect(ctx, req)
if err != nil {
return fmt.Errorf("connect RPC failed: %v", err)
}
if !resp.Success {
return fmt.Errorf("connect failed: %s", resp.Message)
}
log.Printf("Plugin registered with master: %s", resp.MasterId)
return nil
}
// heartbeatLoop sends periodic health reports
func (w *Worker) heartbeatLoop(ctx context.Context) {
ticker := time.NewTicker(w.config.HealthCheckInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-w.done:
return
case <-ticker.C:
w.sendHealthReport(ctx)
}
}
}
// sendHealthReport sends a health report to the admin
func (w *Worker) sendHealthReport(ctx context.Context) {
report := &plugin_pb.HealthReport{
PluginId: w.config.WorkerID,
TimestampMs: time.Now().UnixMilli(),
Status: plugin_pb.HealthStatus_HEALTH_STATUS_HEALTHY,
ActiveJobs: int32(len(w.activeJobs)),
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
_, err := w.pluginClient.ReportHealth(ctx, report)
if err != nil {
log.Printf("Failed to send health report: %v", err)
}
}
// ExecuteDetection performs detection for EC candidates
func (w *Worker) ExecuteDetection(ctx context.Context, volumeMetrics map[uint32]*VolumeMetric) ([]*CandidateVolume, error) {
return w.detector.DetectJobs(volumeMetrics)
}
// ExecuteJob executes an encoding job
func (w *Worker) ExecuteJob(ctx context.Context, jobID string, payload *plugin_pb.JobPayload) error {
req := &plugin_pb.ExecuteJobRequest{
JobId: jobID,
JobType: "encode_volume",
Payload: payload,
RetryCount: 0,
}
w.activeJobs[jobID] = req
defer delete(w.activeJobs, jobID)
// Execute the job
result, err := w.executor.ExecuteJob(req)
if err != nil {
log.Printf("Job execution failed: %v", err)
return err
}
if result.Success {
log.Printf("Job %s completed successfully", jobID)
return w.submitResult(ctx, jobID, result)
}
log.Printf("Job %s failed: %s", jobID, result.ErrorMessage)
return fmt.Errorf("%s", result.ErrorMessage)
}
// submitResult submits job results to admin
func (w *Worker) submitResult(ctx context.Context, jobID string, result *ExecutionResult) error {
jobResult := &plugin_pb.JobResult{
Success: result.Success,
Metadata: result.Metadata,
}
req := &plugin_pb.JobResultRequest{
JobId: jobID,
JobType: "encode_volume",
Status: plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED,
Message: "Encoding completed successfully",
Result: jobResult,
RetryCountUsed: 0,
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
_, err := w.pluginClient.SubmitResult(ctx, req)
return err
}
// Stop gracefully stops the worker
func (w *Worker) Stop(ctx context.Context) error {
log.Printf("Stopping EC worker")
w.isRunning = false
close(w.done)
if w.conn != nil {
return w.conn.Close()
}
return nil
}
// GetStatus returns the current worker status
func (w *Worker) GetStatus() map[string]interface{} {
return map[string]interface{}{
"worker_id": w.config.WorkerID,
"is_running": w.isRunning,
"active_jobs": len(w.activeJobs),
"admin_connected": w.conn != nil,
}
}
// ParseFlags parses command line flags for EC worker
func ParseFlags() *WorkerConfig {
config := &WorkerConfig{
WorkerID: "ec-worker-1",
AdminHost: "localhost",
AdminPort: 50051,
PluginPort: 50052,
StripeSize: 10,
EncodeCopies: 1,
RackAwareness: true,
DataCenterAwareness: false,
MinVolumeSize: 1000,
MaxVolumeSize: 10000,
DetectionInterval: 1 * time.Hour,
MaxConcurrentJobs: 5,
HealthCheckInterval: 30 * time.Second,
RetryPolicy: "exponential",
}
flag.StringVar(&config.WorkerID, "worker-id", config.WorkerID, "Worker ID")
flag.StringVar(&config.AdminHost, "admin-host", config.AdminHost, "Admin server host")
flag.IntVar(&config.AdminPort, "admin-port", config.AdminPort, "Admin server port")
flag.IntVar(&config.PluginPort, "plugin-port", config.PluginPort, "Plugin server port")
flag.IntVar(&config.StripeSize, "stripe-size", config.StripeSize, "Stripe size in MB")
flag.IntVar(&config.EncodeCopies, "encode-copies", config.EncodeCopies, "Copies after encoding")
flag.BoolVar(&config.RackAwareness, "rack-awareness", config.RackAwareness, "Enable rack awareness")
flag.BoolVar(&config.DataCenterAwareness, "dc-awareness", config.DataCenterAwareness, "Enable data center awareness")
flag.Uint64Var(&config.MinVolumeSize, "min-volume-size", config.MinVolumeSize, "Minimum volume size in MB")
flag.Uint64Var(&config.MaxVolumeSize, "max-volume-size", config.MaxVolumeSize, "Maximum volume size in MB")
flag.DurationVar(&config.DetectionInterval, "detection-interval", config.DetectionInterval, "Detection interval")
flag.IntVar(&config.MaxConcurrentJobs, "max-concurrent-jobs", config.MaxConcurrentJobs, "Max concurrent jobs")
flag.DurationVar(&config.HealthCheckInterval, "health-check-interval", config.HealthCheckInterval, "Health check interval")
flag.StringVar(&config.RetryPolicy, "retry-policy", config.RetryPolicy, "Retry policy")
flag.Parse()
return config
}
// ListenAndServe starts the gRPC server for the worker
func (w *Worker) ListenAndServe(port int) error {
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
return fmt.Errorf("failed to listen on port %d: %v", port, err)
}
server := grpc.NewServer()
// Register plugin service handlers here
// plugin_pb.RegisterPluginServiceServer(server, w)
log.Printf("Worker listening on port %d", port)
return server.Serve(listener)
}
@@ -1,166 +0,0 @@
package vacuum
import (
"fmt"
)
// VacuumCandidate represents a volume eligible for vacuum
type VacuumCandidate struct {
VolumeID uint32
DataNodeID string
Size uint64
UsedSpace uint64
DeadSpace uint64
DeadSpacePercent float32
FragmentationScore float32
RackID string
CanVacuum bool
Reason string
}
// DetectionOptions contains options for detection
type DetectionOptions struct {
MinVolumeSize uint64
MaxVolumeSize uint64
DeadSpaceThreshold float32
PreferredNodes []string
ExcludeNodes []string
}
// Detector scans for vacuum candidates
type Detector struct {
config DetectionOptions
}
// NewDetector creates a new vacuum detector
func NewDetector(opts DetectionOptions) *Detector {
return &Detector{
config: opts,
}
}
// DetectJobs scans volumes for vacuum candidates
func (d *Detector) DetectJobs(volumeMetrics map[uint32]*VolumeMetric) ([]*VacuumCandidate, error) {
candidates := make([]*VacuumCandidate, 0)
for volumeID, metric := range volumeMetrics {
candidate, shouldInclude := d.evaluateVolume(volumeID, metric)
if shouldInclude {
candidates = append(candidates, candidate)
}
}
SortByFragmentation(candidates)
return candidates, nil
}
// evaluateVolume checks if a volume should be vacuumed
func (d *Detector) evaluateVolume(volumeID uint32, metric *VolumeMetric) (*VacuumCandidate, bool) {
candidate := &VacuumCandidate{
VolumeID: volumeID,
DataNodeID: metric.DataNodeID,
Size: metric.Size,
UsedSpace: metric.UsedSpace,
RackID: metric.RackID,
}
// Check size constraints
if metric.Size < d.config.MinVolumeSize {
candidate.CanVacuum = false
candidate.Reason = fmt.Sprintf("volume too small: %d < %d", metric.Size, d.config.MinVolumeSize)
return candidate, false
}
if metric.Size > d.config.MaxVolumeSize {
candidate.CanVacuum = false
candidate.Reason = fmt.Sprintf("volume too large: %d > %d", metric.Size, d.config.MaxVolumeSize)
return candidate, false
}
// Calculate dead space
deadSpace := metric.Size - metric.UsedSpace
deadSpacePercent := float32(deadSpace) * 100 / float32(metric.Size)
candidate.DeadSpace = deadSpace
candidate.DeadSpacePercent = deadSpacePercent
// Check dead space threshold
if deadSpacePercent < d.config.DeadSpaceThreshold {
candidate.CanVacuum = false
candidate.Reason = fmt.Sprintf("insufficient dead space: %.2f%% < %.2f%%", deadSpacePercent, d.config.DeadSpaceThreshold)
return candidate, false
}
// Check node exclusion
if d.isNodeExcluded(metric.DataNodeID) {
candidate.CanVacuum = false
candidate.Reason = "node is in exclusion list"
return candidate, false
}
// Check node preference
if len(d.config.PreferredNodes) > 0 && !d.isPreferredNode(metric.DataNodeID) {
candidate.CanVacuum = false
candidate.Reason = "node not in preferred list"
return candidate, false
}
// Calculate fragmentation score
candidate.FragmentationScore = d.calculateFragmentationScore(metric)
candidate.CanVacuum = true
candidate.Reason = "eligible for vacuum"
return candidate, true
}
// isNodeExcluded checks if a node is in the exclusion list
func (d *Detector) isNodeExcluded(nodeID string) bool {
for _, excluded := range d.config.ExcludeNodes {
if excluded == nodeID {
return true
}
}
return false
}
// isPreferredNode checks if a node is in the preferred list
func (d *Detector) isPreferredNode(nodeID string) bool {
for _, preferred := range d.config.PreferredNodes {
if preferred == nodeID {
return true
}
}
return false
}
// calculateFragmentationScore calculates how fragmented a volume is
func (d *Detector) calculateFragmentationScore(metric *VolumeMetric) float32 {
if metric.Size == 0 {
return 0
}
deadSpace := metric.Size - metric.UsedSpace
return float32(deadSpace) * 100 / float32(metric.Size)
}
// VolumeMetric contains volume statistics
type VolumeMetric struct {
VolumeID uint32
DataNodeID string
Size uint64
UsedSpace uint64
FileCount int64
LastVacuumTime int64
RackID string
Collection string
}
// SortByFragmentation sorts candidates by fragmentation score
func SortByFragmentation(candidates []*VacuumCandidate) {
// Simple bubble sort for demonstration
for i := 0; i < len(candidates); i++ {
for j := i + 1; j < len(candidates); j++ {
if candidates[j].FragmentationScore > candidates[i].FragmentationScore {
candidates[i], candidates[j] = candidates[j], candidates[i]
}
}
}
}
@@ -1,247 +0,0 @@
package vacuum
import (
"fmt"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// ExecutionStatus tracks job execution status
type ExecutionStatus string
const (
StatusAnalyzing ExecutionStatus = "analyzing"
StatusDefragmenting ExecutionStatus = "defragmenting"
StatusOptimizing ExecutionStatus = "optimizing"
StatusVerifying ExecutionStatus = "verifying"
StatusCompleted ExecutionStatus = "completed"
StatusFailed ExecutionStatus = "failed"
)
// ExecutionStep represents a step in the vacuum pipeline
type ExecutionStep struct {
Name string
Status ExecutionStatus
StartTime *time.Time
EndTime *time.Time
Progress float32
ErrorMsg string
}
// Executor handles vacuum execution
type Executor struct {
config *ExecutorConfig
}
// ExecutorConfig contains executor configuration
type ExecutorConfig struct {
MinVolumeSize uint64
MaxVolumeSize uint64
TimeoutPerStep time.Duration
MaxRetries int
}
// NewExecutor creates a new vacuum executor
func NewExecutor(config *ExecutorConfig) *Executor {
if config == nil {
config = &ExecutorConfig{
MinVolumeSize: 500,
MaxVolumeSize: 5000,
TimeoutPerStep: 3 * time.Minute,
MaxRetries: 3,
}
}
return &Executor{config: config}
}
// VacuumExecutionResult contains the result of vacuum operation
type VacuumExecutionResult struct {
VolumeID uint32
Success bool
StartTime time.Time
EndTime time.Time
TotalDuration time.Duration
SpaceFreed uint64
FilesMoved int64
FragmentsBefore int64
FragmentsAfter int64
Metadata map[string]string
Steps []*ExecutionStep
ErrorMessage string
}
// ExecuteJob executes the vacuum operation for a volume
func (e *Executor) ExecuteJob(job *plugin_pb.ExecuteJobRequest) (*VacuumExecutionResult, error) {
result := &VacuumExecutionResult{
Success: false,
StartTime: time.Now(),
Metadata: make(map[string]string),
Steps: make([]*ExecutionStep, 0),
}
// Extract volume ID from payload
volumeID := extractVolumeID(job.Payload)
result.VolumeID = volumeID
// Step 1: Analyze fragmentation
if err := e.analyzeFragmentation(result); err != nil {
result.ErrorMessage = fmt.Sprintf("analysis failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 2: Defragment volume
if err := e.defragmentVolume(result); err != nil {
result.ErrorMessage = fmt.Sprintf("defragment failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 3: Optimize storage
if err := e.optimizeStorage(result); err != nil {
result.ErrorMessage = fmt.Sprintf("optimize failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
// Step 4: Verify result
if err := e.verifyResult(result); err != nil {
result.ErrorMessage = fmt.Sprintf("verification failed: %v", err)
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, err
}
result.Success = true
result.EndTime = time.Now()
result.TotalDuration = result.EndTime.Sub(result.StartTime)
return result, nil
}
// analyzeFragmentation analyzes the volume fragmentation
func (e *Executor) analyzeFragmentation(result *VacuumExecutionResult) error {
step := &ExecutionStep{
Name: "analyzing",
Status: StatusAnalyzing,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
// Simulate analysis
time.Sleep(50 * time.Millisecond)
result.FragmentsBefore = 1500
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// defragmentVolume defragments the volume
func (e *Executor) defragmentVolume(result *VacuumExecutionResult) error {
step := &ExecutionStep{
Name: "defragmenting",
Status: StatusDefragmenting,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
// Simulate defragmentation with progress
for i := 0; i < 10; i++ {
time.Sleep(20 * time.Millisecond)
step.Progress = float32((i + 1) * 10)
}
result.FilesMoved = 850
result.SpaceFreed = 1000000
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// optimizeStorage optimizes storage layout
func (e *Executor) optimizeStorage(result *VacuumExecutionResult) error {
step := &ExecutionStep{
Name: "optimizing",
Status: StatusOptimizing,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
// Simulate optimization
for i := 0; i < 5; i++ {
time.Sleep(30 * time.Millisecond)
step.Progress = float32((i + 1) * 20)
}
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// verifyResult verifies the operation result
func (e *Executor) verifyResult(result *VacuumExecutionResult) error {
step := &ExecutionStep{
Name: "verifying",
Status: StatusVerifying,
Progress: 0,
}
now := time.Now()
step.StartTime = &now
// Simulate verification
time.Sleep(50 * time.Millisecond)
result.FragmentsAfter = 320
result.Metadata["space_freed_mb"] = "1"
result.Metadata["files_moved"] = "850"
result.Metadata["fragmentation_reduction"] = "78.7%"
step.Progress = 100
step.EndTime = &now
result.Steps = append(result.Steps, step)
return nil
}
// extractVolumeID extracts the volume ID from job payload
func extractVolumeID(payload *plugin_pb.JobPayload) uint32 {
if payload == nil || len(payload.Data) < 4 {
return 0
}
return uint32(payload.Data[0]) |
(uint32(payload.Data[1]) << 8) |
(uint32(payload.Data[2]) << 16) |
(uint32(payload.Data[3]) << 24)
}
// ValidateExecutionResult validates the result of execution
func ValidateExecutionResult(result *VacuumExecutionResult) bool {
if !result.Success {
return false
}
if result.EndTime.Before(result.StartTime) {
return false
}
if len(result.Steps) != 4 {
return false
}
return true
}
-178
View File
@@ -1,178 +0,0 @@
package vacuum
import (
"encoding/json"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// ConfigurationSchema defines the schema for vacuum plugin configuration
type ConfigurationSchema struct {
AdminConfig AdminConfigSchema `json:"admin_config"`
WorkerConfig WorkerConfigSchema `json:"worker_config"`
}
// AdminConfigSchema defines admin-side configuration
type AdminConfigSchema struct {
VacuumInterval ConfigField `json:"vacuum_interval"`
MaxConcurrentJobs ConfigField `json:"max_concurrent_jobs"`
JobTimeout ConfigField `json:"job_timeout"`
HealthCheckInterval ConfigField `json:"health_check_interval"`
DeadSpaceThreshold ConfigField `json:"dead_space_threshold"`
}
// WorkerConfigSchema defines worker-side configuration
type WorkerConfigSchema struct {
MinVolumeSize ConfigField `json:"min_volume_size"`
MaxVolumeSize ConfigField `json:"max_volume_size"`
TargetUtilization ConfigField `json:"target_utilization"`
BatchSize ConfigField `json:"batch_size"`
}
// ConfigField describes a configuration field
type ConfigField struct {
Name string `json:"name"`
Description string `json:"description"`
Type string `json:"type"`
Required bool `json:"required"`
Default interface{} `json:"default,omitempty"`
Min interface{} `json:"min,omitempty"`
Max interface{} `json:"max,omitempty"`
Options []interface{} `json:"options,omitempty"`
Unit string `json:"unit,omitempty"`
}
// GetConfigurationSchema returns the schema for vacuum plugin configuration
func GetConfigurationSchema() *plugin_pb.PluginConfig {
schema := ConfigurationSchema{
AdminConfig: AdminConfigSchema{
VacuumInterval: ConfigField{
Name: "vacuum_interval",
Description: "Time between vacuum operations",
Type: "duration",
Required: true,
Default: "4h",
Min: "1h",
Max: "24h",
Unit: "seconds",
},
MaxConcurrentJobs: ConfigField{
Name: "max_concurrent_jobs",
Description: "Maximum concurrent vacuum jobs",
Type: "integer",
Required: true,
Default: 3,
Min: 1,
Max: 10,
},
JobTimeout: ConfigField{
Name: "job_timeout",
Description: "Timeout for individual vacuum jobs",
Type: "duration",
Required: true,
Default: "8h",
Min: "1h",
Max: "24h",
Unit: "seconds",
},
HealthCheckInterval: ConfigField{
Name: "health_check_interval",
Description: "Health check interval",
Type: "duration",
Required: true,
Default: "30s",
Min: "5s",
Max: "5m",
Unit: "seconds",
},
DeadSpaceThreshold: ConfigField{
Name: "dead_space_threshold",
Description: "Dead space percentage threshold for vacuum",
Type: "integer",
Required: true,
Default: 30,
Min: 5,
Max: 95,
Unit: "percent",
},
},
WorkerConfig: WorkerConfigSchema{
MinVolumeSize: ConfigField{
Name: "min_volume_size",
Description: "Minimum volume size to consider for vacuum",
Type: "integer",
Required: true,
Default: 500,
Min: 100,
Unit: "MB",
},
MaxVolumeSize: ConfigField{
Name: "max_volume_size",
Description: "Maximum volume size to consider for vacuum",
Type: "integer",
Required: true,
Default: 5000,
Max: 50000,
Unit: "MB",
},
TargetUtilization: ConfigField{
Name: "target_utilization",
Description: "Target volume utilization after vacuum",
Type: "integer",
Required: true,
Default: 80,
Min: 50,
Max: 95,
Unit: "percent",
},
BatchSize: ConfigField{
Name: "batch_size",
Description: "Number of volumes to process in a batch",
Type: "integer",
Required: true,
Default: 10,
Min: 1,
Max: 100,
},
},
}
data, _ := json.MarshalIndent(schema, "", " ")
return &plugin_pb.PluginConfig{
PluginId: "vacuum-plugin",
Properties: map[string]string{
"schema": string(data),
"vacuum_interval": "4h",
"max_concurrent_jobs": "3",
"job_timeout": "8h",
"health_check_interval": "30s",
"dead_space_threshold": "30",
"min_volume_size": "500",
"max_volume_size": "5000",
"target_utilization": "80",
"batch_size": "10",
},
}
}
// DefaultAdminConfig returns default admin configuration
func DefaultAdminConfig() map[string]string {
return map[string]string{
"vacuum_interval": "4h",
"max_concurrent_jobs": "3",
"job_timeout": "8h",
"health_check_interval": "30s",
"dead_space_threshold": "30",
}
}
// DefaultWorkerConfig returns default worker configuration
func DefaultWorkerConfig() map[string]string {
return map[string]string{
"min_volume_size": "500",
"max_volume_size": "5000",
"target_utilization": "80",
"batch_size": "10",
}
}
-330
View File
@@ -1,330 +0,0 @@
package vacuum
import (
"context"
"flag"
"fmt"
"log"
"net"
"time"
"google.golang.org/grpc"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
)
// WorkerConfig holds worker-specific configuration
type WorkerConfig struct {
WorkerID string
AdminHost string
AdminPort int
PluginPort int
MinVolumeSize uint64
MaxVolumeSize uint64
TargetUtilization int
BatchSize int
VacuumInterval time.Duration
MaxConcurrentJobs int
HealthCheckInterval time.Duration
DeadSpaceThreshold int
}
// Worker represents the vacuum plugin worker
type Worker struct {
config *WorkerConfig
pluginClient plugin_pb.PluginServiceClient
conn *grpc.ClientConn
detector *Detector
executor *Executor
activeJobs map[string]*plugin_pb.ExecuteJobRequest
done chan bool
isRunning bool
}
// NewWorker creates a new vacuum worker
func NewWorker(config *WorkerConfig) *Worker {
return &Worker{
config: config,
activeJobs: make(map[string]*plugin_pb.ExecuteJobRequest),
done: make(chan bool),
}
}
// Start initializes and starts the worker
func (w *Worker) Start(ctx context.Context) error {
log.Printf("Starting vacuum worker: %s", w.config.WorkerID)
// Connect to admin server
if err := w.connectToAdmin(ctx); err != nil {
return fmt.Errorf("failed to connect to admin: %v", err)
}
// Initialize detector
w.detector = NewDetector(DetectionOptions{
MinVolumeSize: w.config.MinVolumeSize,
MaxVolumeSize: w.config.MaxVolumeSize,
DeadSpaceThreshold: float32(w.config.DeadSpaceThreshold),
})
// Initialize executor
w.executor = NewExecutor(&ExecutorConfig{
MinVolumeSize: w.config.MinVolumeSize,
MaxVolumeSize: w.config.MaxVolumeSize,
TimeoutPerStep: 3 * time.Minute,
MaxRetries: 3,
})
// Register with admin
if err := w.registerPlugin(ctx); err != nil {
return fmt.Errorf("failed to register: %v", err)
}
w.isRunning = true
// Start background goroutines
go w.heartbeatLoop(ctx)
log.Printf("Vacuum worker started successfully")
return nil
}
// connectToAdmin establishes connection to admin server
func (w *Worker) connectToAdmin(ctx context.Context) error {
address := fmt.Sprintf("%s:%d", w.config.AdminHost, w.config.AdminPort)
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
conn, err := grpc.DialContext(dialCtx, address, grpc.WithInsecure())
if err != nil {
return fmt.Errorf("failed to dial: %v", err)
}
w.conn = conn
w.pluginClient = plugin_pb.NewPluginServiceClient(conn)
return nil
}
// registerPlugin registers the plugin with the admin server
func (w *Worker) registerPlugin(ctx context.Context) error {
schema := GetConfigurationSchema()
req := &plugin_pb.PluginConnectRequest{
PluginId: w.config.WorkerID,
PluginName: "vacuum-plugin",
Version: "1.0.0",
Capabilities: []string{"detect", "execute", "report_health"},
MaxConcurrentJobs: int32(w.config.MaxConcurrentJobs),
SupportsStreaming: true,
Port: int32(w.config.PluginPort),
}
// Add capabilities detail
req.CapabilitiesDetail = &plugin_pb.PluginCapabilities{
Detection: []*plugin_pb.DetectionCapability{
{
Type: "vacuum_candidates",
Description: "Detect volumes eligible for vacuum",
MinIntervalSeconds: int32(w.config.VacuumInterval.Seconds()),
RequiresFullScan: false,
},
},
Maintenance: []*plugin_pb.MaintenanceCapability{
{
Type: "vacuum_volume",
Description: "Vacuum a volume to free dead space",
RequiredDetectionTypes: []string{"vacuum_candidates"},
EstimatedDurationSeconds: 1800,
},
},
}
// Add schema to metadata
if schema != nil {
if req.Metadata == nil {
req.Metadata = make(map[string]string)
}
for k, v := range schema.Properties {
req.Metadata[k] = v
}
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
resp, err := w.pluginClient.Connect(ctx, req)
if err != nil {
return fmt.Errorf("connect RPC failed: %v", err)
}
if !resp.Success {
return fmt.Errorf("connect failed: %s", resp.Message)
}
log.Printf("Plugin registered with master: %s", resp.MasterId)
return nil
}
// heartbeatLoop sends periodic health reports
func (w *Worker) heartbeatLoop(ctx context.Context) {
ticker := time.NewTicker(w.config.HealthCheckInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-w.done:
return
case <-ticker.C:
w.sendHealthReport(ctx)
}
}
}
// sendHealthReport sends a health report to the admin
func (w *Worker) sendHealthReport(ctx context.Context) {
report := &plugin_pb.HealthReport{
PluginId: w.config.WorkerID,
TimestampMs: time.Now().UnixMilli(),
Status: plugin_pb.HealthStatus_HEALTH_STATUS_HEALTHY,
ActiveJobs: int32(len(w.activeJobs)),
}
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
_, err := w.pluginClient.ReportHealth(ctx, report)
if err != nil {
log.Printf("Failed to send health report: %v", err)
}
}
// ExecuteDetection performs detection for vacuum candidates
func (w *Worker) ExecuteDetection(ctx context.Context, volumeMetrics map[uint32]*VolumeMetric) ([]*VacuumCandidate, error) {
return w.detector.DetectJobs(volumeMetrics)
}
// ExecuteJob executes a vacuum job
func (w *Worker) ExecuteJob(ctx context.Context, jobID string, payload *plugin_pb.JobPayload) error {
req := &plugin_pb.ExecuteJobRequest{
JobId: jobID,
JobType: "vacuum_volume",
Payload: payload,
RetryCount: 0,
}
w.activeJobs[jobID] = req
defer delete(w.activeJobs, jobID)
// Execute the job
result, err := w.executor.ExecuteJob(req)
if err != nil {
log.Printf("Job execution failed: %v", err)
return err
}
if result.Success {
log.Printf("Job %s completed successfully", jobID)
return w.submitResult(ctx, jobID, result)
}
log.Printf("Job %s failed: %s", jobID, result.ErrorMessage)
return fmt.Errorf("%s", result.ErrorMessage)
}
// submitResult submits job results to admin
func (w *Worker) submitResult(ctx context.Context, jobID string, result *VacuumExecutionResult) error {
jobResult := &plugin_pb.JobResult{
Success: result.Success,
Metadata: result.Metadata,
}
req := &plugin_pb.JobResultRequest{
JobId: jobID,
JobType: "vacuum_volume",
Status: plugin_pb.ExecutionStatus_EXECUTION_STATUS_COMPLETED,
Message: "Vacuum completed successfully",
Result: jobResult,
RetryCountUsed: 0,
}
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
_, err := w.pluginClient.SubmitResult(ctx, req)
return err
}
// Stop gracefully stops the worker
func (w *Worker) Stop(ctx context.Context) error {
log.Printf("Stopping vacuum worker")
w.isRunning = false
close(w.done)
if w.conn != nil {
return w.conn.Close()
}
return nil
}
// GetStatus returns the current worker status
func (w *Worker) GetStatus() map[string]interface{} {
return map[string]interface{}{
"worker_id": w.config.WorkerID,
"is_running": w.isRunning,
"active_jobs": len(w.activeJobs),
"admin_connected": w.conn != nil,
}
}
// ParseFlags parses command line flags for vacuum worker
func ParseFlags() *WorkerConfig {
config := &WorkerConfig{
WorkerID: "vacuum-worker-1",
AdminHost: "localhost",
AdminPort: 50051,
PluginPort: 50053,
MinVolumeSize: 500,
MaxVolumeSize: 5000,
TargetUtilization: 80,
BatchSize: 10,
VacuumInterval: 4 * time.Hour,
MaxConcurrentJobs: 3,
HealthCheckInterval: 30 * time.Second,
DeadSpaceThreshold: 30,
}
flag.StringVar(&config.WorkerID, "worker-id", config.WorkerID, "Worker ID")
flag.StringVar(&config.AdminHost, "admin-host", config.AdminHost, "Admin server host")
flag.IntVar(&config.AdminPort, "admin-port", config.AdminPort, "Admin server port")
flag.IntVar(&config.PluginPort, "plugin-port", config.PluginPort, "Plugin server port")
flag.Uint64Var(&config.MinVolumeSize, "min-volume-size", config.MinVolumeSize, "Minimum volume size in MB")
flag.Uint64Var(&config.MaxVolumeSize, "max-volume-size", config.MaxVolumeSize, "Maximum volume size in MB")
flag.IntVar(&config.TargetUtilization, "target-utilization", config.TargetUtilization, "Target utilization percentage")
flag.IntVar(&config.BatchSize, "batch-size", config.BatchSize, "Batch size")
flag.DurationVar(&config.VacuumInterval, "vacuum-interval", config.VacuumInterval, "Vacuum interval")
flag.IntVar(&config.MaxConcurrentJobs, "max-concurrent-jobs", config.MaxConcurrentJobs, "Max concurrent jobs")
flag.DurationVar(&config.HealthCheckInterval, "health-check-interval", config.HealthCheckInterval, "Health check interval")
flag.IntVar(&config.DeadSpaceThreshold, "dead-space-threshold", config.DeadSpaceThreshold, "Dead space threshold percentage")
flag.Parse()
return config
}
// ListenAndServe starts the gRPC server for the worker
func (w *Worker) ListenAndServe(port int) error {
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
return fmt.Errorf("failed to listen on port %d: %v", port, err)
}
server := grpc.NewServer()
log.Printf("Worker listening on port %d", port)
return server.Serve(listener)
}
-122
View File
@@ -1,122 +0,0 @@
package app
type PluginConfigPageData struct {
JobType string
Config JobTypeConfig
DetectionHistory []interface{}
ExecutionHistory []interface{}
ActiveTab string
}
type JobTypeConfig struct {
Type string
Enabled bool
Priority int
Interval int64
MaxConcurrent int
Parameters map[string]string
RequiredDetections []string
}
templ PluginConfiguration(data PluginConfigPageData) {
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
<h1 class="h2">
<i class="fas fa-cog me-2"></i>Configuration
</h1>
<div class="btn-toolbar mb-2 mb-md-0">
<div class="btn-group me-2">
<a href="/plugins" class="btn btn-sm btn-outline-secondary">
<i class="fas fa-arrow-left me-1"></i>Back
</a>
</div>
</div>
</div>
<div class="card shadow mb-4">
<div class="card-header py-3">
<h6 class="m-0 font-weight-bold text-primary">Configuration</h6>
</div>
<div class="card-body">
<form id="configForm">
<div class="mb-3">
<label class="form-label">Enabled</label>
<div class="form-check">
<input class="form-check-input" type="checkbox" id="enabled" />
<label class="form-check-label" for="enabled">Enable this job type</label>
</div>
</div>
<button type="button" class="btn btn-primary" id="saveBtn">Save Configuration</button>
</form>
</div>
</div>
<div class="card shadow mb-4">
<div class="card-header py-3">
<h6 class="m-0 font-weight-bold text-primary">Detection History</h6>
</div>
<div class="card-body">
if len(data.DetectionHistory) == 0 {
<div class="alert alert-info">No detection records</div>
} else {
<div class="table-responsive">
<table class="table table-hover table-sm">
<thead>
<tr>
<th>Timestamp</th>
<th>Type</th>
<th>Severity</th>
</tr>
</thead>
<tbody>
for i := 0; i < len(data.DetectionHistory); i++ {
<tr>
<td>2024-01-01 12:00:00</td>
<td><code>detection</code></td>
<td><span class="badge bg-info">LOW</span></td>
</tr>
}
</tbody>
</table>
</div>
}
</div>
</div>
<div class="card shadow mb-4">
<div class="card-header py-3">
<h6 class="m-0 font-weight-bold text-primary">Execution History</h6>
</div>
<div class="card-body">
if len(data.ExecutionHistory) == 0 {
<div class="alert alert-info">No execution records</div>
} else {
<div class="table-responsive">
<table class="table table-hover table-sm">
<thead>
<tr>
<th>Job ID</th>
<th>State</th>
<th>Created</th>
</tr>
</thead>
<tbody>
for i := 0; i < len(data.ExecutionHistory); i++ {
<tr>
<td><code>job_id</code></td>
<td><span class="badge bg-success">COMPLETED</span></td>
<td>2024-01-01 12:00:00</td>
</tr>
}
</tbody>
</table>
</div>
}
</div>
</div>
<script>
document.getElementById('saveBtn').addEventListener('click', function() {
alert('Configuration saved');
});
</script>
}
-108
View File
@@ -1,108 +0,0 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.977
package app
//lint:file-ignore SA4006 This context is only used if a nested component is present.
import "github.com/a-h/templ"
import templruntime "github.com/a-h/templ/runtime"
type PluginConfigPageData struct {
JobType string
Config JobTypeConfig
DetectionHistory []interface{}
ExecutionHistory []interface{}
ActiveTab string
}
type JobTypeConfig struct {
Type string
Enabled bool
Priority int
Interval int64
MaxConcurrent int
Parameters map[string]string
RequiredDetections []string
}
func PluginConfiguration(data PluginConfigPageData) templ.Component {
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
return templ_7745c5c3_CtxErr
}
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
if !templ_7745c5c3_IsBuffer {
defer func() {
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
if templ_7745c5c3_Err == nil {
templ_7745c5c3_Err = templ_7745c5c3_BufErr
}
}()
}
ctx = templ.InitializeContext(ctx)
templ_7745c5c3_Var1 := templ.GetChildren(ctx)
if templ_7745c5c3_Var1 == nil {
templ_7745c5c3_Var1 = templ.NopComponent
}
ctx = templ.ClearChildren(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><h1 class=\"h2\"><i class=\"fas fa-cog me-2\"></i>Configuration</h1><div class=\"btn-toolbar mb-2 mb-md-0\"><div class=\"btn-group me-2\"><a href=\"/plugins\" class=\"btn btn-sm btn-outline-secondary\"><i class=\"fas fa-arrow-left me-1\"></i>Back</a></div></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\">Configuration</h6></div><div class=\"card-body\"><form id=\"configForm\"><div class=\"mb-3\"><label class=\"form-label\">Enabled</label><div class=\"form-check\"><input class=\"form-check-input\" type=\"checkbox\" id=\"enabled\"> <label class=\"form-check-label\" for=\"enabled\">Enable this job type</label></div></div><button type=\"button\" class=\"btn btn-primary\" id=\"saveBtn\">Save Configuration</button></form></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\">Detection History</h6></div><div class=\"card-body\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if len(data.DetectionHistory) == 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "<div class=\"alert alert-info\">No detection records</div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "<div class=\"table-responsive\"><table class=\"table table-hover table-sm\"><thead><tr><th>Timestamp</th><th>Type</th><th>Severity</th></tr></thead> <tbody>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
for i := 0; i < len(data.DetectionHistory); i++ {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "<tr><td>2024-01-01 12:00:00</td><td><code>detection</code></td><td><span class=\"badge bg-info\">LOW</span></td></tr>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</tbody></table></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\">Execution History</h6></div><div class=\"card-body\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if len(data.ExecutionHistory) == 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "<div class=\"alert alert-info\">No execution records</div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "<div class=\"table-responsive\"><table class=\"table table-hover table-sm\"><thead><tr><th>Job ID</th><th>State</th><th>Created</th></tr></thead> <tbody>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
for i := 0; i < len(data.ExecutionHistory); i++ {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "<tr><td><code>job_id</code></td><td><span class=\"badge bg-success\">COMPLETED</span></td><td>2024-01-01 12:00:00</td></tr>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</tbody></table></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</div></div><script>\ndocument.getElementById('saveBtn').addEventListener('click', function() {\nalert('Configuration saved');\n});\n</script>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
return nil
})
}
var _ = templruntime.GeneratedTemplate
-65
View File
@@ -1,65 +0,0 @@
package app
type PluginJobsPageData struct {
JobType string
Jobs []interface{}
StateFilter string
}
templ PluginJobsMonitoring(data PluginJobsPageData) {
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
<h1 class="h2">
<i class="fas fa-tasks me-2"></i>Jobs
</h1>
<div class="btn-toolbar mb-2 mb-md-0">
<div class="btn-group me-2">
<button type="button" class="btn btn-sm btn-success" id="triggerBtn">
<i class="fas fa-play me-1"></i>Trigger Detection
</button>
<a href="/plugins" class="btn btn-sm btn-outline-secondary">
<i class="fas fa-arrow-left me-1"></i>Back
</a>
</div>
</div>
</div>
<div class="card shadow mb-4">
<div class="card-header py-3">
<h6 class="m-0 font-weight-bold text-primary">Recent Jobs</h6>
</div>
<div class="card-body">
if len(data.Jobs) == 0 {
<div class="alert alert-info">No jobs found</div>
} else {
<div class="table-responsive">
<table class="table table-hover table-sm">
<thead>
<tr>
<th>Job ID</th>
<th>State</th>
<th>Created</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
for i := 0; i < len(data.Jobs); i++ {
<tr>
<td><code>job_id</code></td>
<td><span class="badge bg-secondary">PENDING</span></td>
<td>2024-01-01 12:00:00</td>
<td></td>
</tr>
}
</tbody>
</table>
</div>
}
</div>
</div>
<script>
document.getElementById('triggerBtn').addEventListener('click', function() {
alert('Triggering detection');
});
</script>
}
-71
View File
@@ -1,71 +0,0 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.977
package app
//lint:file-ignore SA4006 This context is only used if a nested component is present.
import "github.com/a-h/templ"
import templruntime "github.com/a-h/templ/runtime"
type PluginJobsPageData struct {
JobType string
Jobs []interface{}
StateFilter string
}
func PluginJobsMonitoring(data PluginJobsPageData) templ.Component {
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
return templ_7745c5c3_CtxErr
}
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
if !templ_7745c5c3_IsBuffer {
defer func() {
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
if templ_7745c5c3_Err == nil {
templ_7745c5c3_Err = templ_7745c5c3_BufErr
}
}()
}
ctx = templ.InitializeContext(ctx)
templ_7745c5c3_Var1 := templ.GetChildren(ctx)
if templ_7745c5c3_Var1 == nil {
templ_7745c5c3_Var1 = templ.NopComponent
}
ctx = templ.ClearChildren(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><h1 class=\"h2\"><i class=\"fas fa-tasks me-2\"></i>Jobs</h1><div class=\"btn-toolbar mb-2 mb-md-0\"><div class=\"btn-group me-2\"><button type=\"button\" class=\"btn btn-sm btn-success\" id=\"triggerBtn\"><i class=\"fas fa-play me-1\"></i>Trigger Detection</button> <a href=\"/plugins\" class=\"btn btn-sm btn-outline-secondary\"><i class=\"fas fa-arrow-left me-1\"></i>Back</a></div></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\">Recent Jobs</h6></div><div class=\"card-body\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if len(data.Jobs) == 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "<div class=\"alert alert-info\">No jobs found</div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "<div class=\"table-responsive\"><table class=\"table table-hover table-sm\"><thead><tr><th>Job ID</th><th>State</th><th>Created</th><th>Actions</th></tr></thead> <tbody>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
for i := 0; i < len(data.Jobs); i++ {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "<tr><td><code>job_id</code></td><td><span class=\"badge bg-secondary\">PENDING</span></td><td>2024-01-01 12:00:00</td><td></td></tr>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "</tbody></table></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</div></div><script>\ndocument.getElementById('triggerBtn').addEventListener('click', function() {\nalert('Triggering detection');\n});\n</script>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
return nil
})
}
var _ = templruntime.GeneratedTemplate
-92
View File
@@ -1,92 +0,0 @@
package app
import "fmt"
type PluginsPageData struct {
Plugins []map[string]interface{}
JobTypes map[string]interface{}
}
templ PluginsOverview(data PluginsPageData) {
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
<h1 class="h2">
<i class="fas fa-plug me-2"></i>Plugins
</h1>
<div class="btn-toolbar mb-2 mb-md-0">
<div class="btn-group me-2">
<a href="/plugins" class="btn btn-sm btn-outline-primary">
<i class="fas fa-sync-alt me-1"></i>Refresh
</a>
</div>
</div>
</div>
<div class="row mb-4">
<div class="col-md-3 mb-4">
<div class="card border-left-primary shadow h-100 py-2">
<div class="card-body">
<div class="row no-gutters align-items-center">
<div class="col mr-2">
<div class="text-xs font-weight-bold text-primary text-uppercase mb-1">Connected Plugins</div>
<div class="h3 mb-0">{ fmt.Sprintf("%d", len(data.Plugins)) }</div>
</div>
<div class="col-auto">
<i class="fas fa-plug fa-2x text-gray-300"></i>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="card shadow mb-4">
<div class="card-header py-3">
<h6 class="m-0 font-weight-bold text-primary">Connected Plugins</h6>
</div>
<div class="card-body">
if len(data.Plugins) == 0 {
<div class="alert alert-info">No plugins connected</div>
} else {
<div class="table-responsive">
<table class="table table-hover table-sm">
<thead>
<tr>
<th>Plugin ID</th>
<th>Name</th>
<th>Status</th>
<th>Version</th>
<th>Capabilities</th>
<th>Actions</th>
</tr>
</thead>
<tbody>
for _, p := range data.Plugins {
<tr>
<td><code>{ p["id"].(string) }</code></td>
<td>{ p["name"].(string) }</td>
<td>
if p["status"].(string) == "CONNECTED" {
<span class="badge bg-success">Connected</span>
} else {
<span class="badge bg-warning">{ p["status"].(string) }</span>
}
</td>
<td>{ p["version"].(string) }</td>
<td>
for _, cap := range p["capabilities"].([]string) {
<span class="badge bg-info me-1">{ cap }</span>
}
</td>
<td>
<a href={ templ.SafeURL("/plugins/jobs/" + p["id"].(string)) } class="btn btn-sm btn-primary">Jobs</a>
<a href={ templ.SafeURL("/plugins/config/" + p["id"].(string)) } class="btn btn-sm btn-secondary">Config</a>
</td>
</tr>
}
</tbody>
</table>
</div>
}
</div>
</div>
}
-201
View File
@@ -1,201 +0,0 @@
// Code generated by templ - DO NOT EDIT.
// templ: version: v0.3.977
package app
//lint:file-ignore SA4006 This context is only used if a nested component is present.
import "github.com/a-h/templ"
import templruntime "github.com/a-h/templ/runtime"
import "fmt"
type PluginsPageData struct {
Plugins []map[string]interface{}
JobTypes map[string]interface{}
}
func PluginsOverview(data PluginsPageData) templ.Component {
return templruntime.GeneratedTemplate(func(templ_7745c5c3_Input templruntime.GeneratedComponentInput) (templ_7745c5c3_Err error) {
templ_7745c5c3_W, ctx := templ_7745c5c3_Input.Writer, templ_7745c5c3_Input.Context
if templ_7745c5c3_CtxErr := ctx.Err(); templ_7745c5c3_CtxErr != nil {
return templ_7745c5c3_CtxErr
}
templ_7745c5c3_Buffer, templ_7745c5c3_IsBuffer := templruntime.GetBuffer(templ_7745c5c3_W)
if !templ_7745c5c3_IsBuffer {
defer func() {
templ_7745c5c3_BufErr := templruntime.ReleaseBuffer(templ_7745c5c3_Buffer)
if templ_7745c5c3_Err == nil {
templ_7745c5c3_Err = templ_7745c5c3_BufErr
}
}()
}
ctx = templ.InitializeContext(ctx)
templ_7745c5c3_Var1 := templ.GetChildren(ctx)
if templ_7745c5c3_Var1 == nil {
templ_7745c5c3_Var1 = templ.NopComponent
}
ctx = templ.ClearChildren(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<div class=\"d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom\"><h1 class=\"h2\"><i class=\"fas fa-plug me-2\"></i>Plugins</h1><div class=\"btn-toolbar mb-2 mb-md-0\"><div class=\"btn-group me-2\"><a href=\"/plugins\" class=\"btn btn-sm btn-outline-primary\"><i class=\"fas fa-sync-alt me-1\"></i>Refresh</a></div></div></div><div class=\"row mb-4\"><div class=\"col-md-3 mb-4\"><div class=\"card border-left-primary shadow h-100 py-2\"><div class=\"card-body\"><div class=\"row no-gutters align-items-center\"><div class=\"col mr-2\"><div class=\"text-xs font-weight-bold text-primary text-uppercase mb-1\">Connected Plugins</div><div class=\"h3 mb-0\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var2 string
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", len(data.Plugins)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/plugins.templ`, Line: 31, Col: 59}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 2, "</div></div><div class=\"col-auto\"><i class=\"fas fa-plug fa-2x text-gray-300\"></i></div></div></div></div></div></div><div class=\"card shadow mb-4\"><div class=\"card-header py-3\"><h6 class=\"m-0 font-weight-bold text-primary\">Connected Plugins</h6></div><div class=\"card-body\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if len(data.Plugins) == 0 {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 3, "<div class=\"alert alert-info\">No plugins connected</div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 4, "<div class=\"table-responsive\"><table class=\"table table-hover table-sm\"><thead><tr><th>Plugin ID</th><th>Name</th><th>Status</th><th>Version</th><th>Capabilities</th><th>Actions</th></tr></thead> <tbody>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
for _, p := range data.Plugins {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 5, "<tr><td><code>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var3 string
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(p["id"].(string))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/plugins.templ`, Line: 65, Col: 28}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 6, "</code></td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var4 string
templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(p["name"].(string))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/plugins.templ`, Line: 66, Col: 24}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 7, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if p["status"].(string) == "CONNECTED" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 8, "<span class=\"badge bg-success\">Connected</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 9, "<span class=\"badge bg-warning\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var5 string
templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(p["status"].(string))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/plugins.templ`, Line: 71, Col: 53}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 10, "</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 11, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(p["version"].(string))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/plugins.templ`, Line: 74, Col: 27}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 12, "</td><td>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
for _, cap := range p["capabilities"].([]string) {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 13, "<span class=\"badge bg-info me-1\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var7 string
templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(cap)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/plugins.templ`, Line: 77, Col: 38}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 14, "</span>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 15, "</td><td><a href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var8 templ.SafeURL
templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL("/plugins/jobs/" + p["id"].(string)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/plugins.templ`, Line: 81, Col: 60}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 16, "\" class=\"btn btn-sm btn-primary\">Jobs</a> <a href=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var9 templ.SafeURL
templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL("/plugins/config/" + p["id"].(string)))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/app/plugins.templ`, Line: 82, Col: 62}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 17, "\" class=\"btn btn-sm btn-secondary\">Config</a></td></tr>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 18, "</tbody></table></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 19, "</div></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
return nil
})
}
var _ = templruntime.GeneratedTemplate
-37
View File
@@ -28,9 +28,6 @@ templ Layout(c *gin.Context, content templ.Component) {
// Detect if we're on a cluster page (but not storage page) to keep submenu expanded
isClusterPage := (strings.HasPrefix(currentPath, "/cluster/masters") || strings.HasPrefix(currentPath, "/cluster/volume-servers") || strings.HasPrefix(currentPath, "/cluster/filers"))
// Detect if we're on a plugins page to keep submenu expanded
isPluginsPage := strings.HasPrefix(currentPath, "/plugins")
}}
<!DOCTYPE html>
<html lang="en">
@@ -333,40 +330,6 @@ templ Layout(c *gin.Context, content templ.Component) {
</a>
}
</li>
<li class="nav-item border-top pt-3 my-3">
if isPluginsPage {
<a class="nav-link active" href="#pluginsSubmenu" data-bs-toggle="collapse">
<i class="fas fa-plug me-2"></i>Plugins
<i class="fas fa-chevron-down ms-auto"></i>
</a>
} else {
<a class="nav-link collapsed" href="#pluginsSubmenu" data-bs-toggle="collapse">
<i class="fas fa-plug me-2"></i>Plugins
<i class="fas fa-chevron-down ms-auto"></i>
</a>
}
if isPluginsPage {
<div class="collapse show" id="pluginsSubmenu">
<ul class="nav flex-column ms-3">
<li class="nav-item">
<a class="nav-link py-2 active" href="/plugins">
<i class="fas fa-home me-2"></i>Overview
</a>
</li>
</ul>
</div>
} else {
<div class="collapse" id="pluginsSubmenu">
<ul class="nav flex-column ms-3">
<li class="nav-item">
<a class="nav-link py-2" href="/plugins">
<i class="fas fa-home me-2"></i>Overview
</a>
</li>
</ul>
</div>
}
</li>
</ul>
</div>
</div>
+27 -56
View File
@@ -55,9 +55,6 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
// Detect if we're on a cluster page (but not storage page) to keep submenu expanded
isClusterPage := (strings.HasPrefix(currentPath, "/cluster/masters") || strings.HasPrefix(currentPath, "/cluster/volume-servers") || strings.HasPrefix(currentPath, "/cluster/filers"))
// Detect if we're on a plugins page to keep submenu expanded
isPluginsPage := strings.HasPrefix(currentPath, "/plugins")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 1, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>SeaweedFS Admin</title><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><meta name=\"csrf-token\" content=\"")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
@@ -65,7 +62,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var2 string
templ_7745c5c3_Var2, templ_7745c5c3_Err = templ.JoinStringErrs(csrfToken)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 41, Col: 47}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 38, Col: 47}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var2))
if templ_7745c5c3_Err != nil {
@@ -78,7 +75,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var3 string
templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(username)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 72, Col: 73}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 69, Col: 73}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3))
if templ_7745c5c3_Err != nil {
@@ -113,7 +110,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var6 string
templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isClusterPage))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 99, Col: 207}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 96, Col: 207}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6))
if templ_7745c5c3_Err != nil {
@@ -170,7 +167,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var11 string
templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%t", isStoragePage))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 124, Col: 207}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 121, Col: 207}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11))
if templ_7745c5c3_Err != nil {
@@ -288,7 +285,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var14 templ.SafeURL
templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 288, Col: 117}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 285, Col: 117}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14))
if templ_7745c5c3_Err != nil {
@@ -323,7 +320,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var17 string
templ_7745c5c3_Var17, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 289, Col: 109}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 286, Col: 109}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var17))
if templ_7745c5c3_Err != nil {
@@ -341,7 +338,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var18 templ.SafeURL
templ_7745c5c3_Var18, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 292, Col: 110}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 289, Col: 110}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var18))
if templ_7745c5c3_Err != nil {
@@ -376,7 +373,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var21 string
templ_7745c5c3_Var21, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 293, Col: 109}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 290, Col: 109}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var21))
if templ_7745c5c3_Err != nil {
@@ -409,7 +406,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var22 templ.SafeURL
templ_7745c5c3_Var22, templ_7745c5c3_Err = templ.JoinURLErrs(templ.SafeURL(menuItem.URL))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 305, Col: 106}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 302, Col: 106}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var22))
if templ_7745c5c3_Err != nil {
@@ -444,7 +441,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
var templ_7745c5c3_Var25 string
templ_7745c5c3_Var25, templ_7745c5c3_Err = templ.JoinStringErrs(menuItem.Name)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 306, Col: 105}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 303, Col: 105}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var25))
if templ_7745c5c3_Err != nil {
@@ -490,33 +487,7 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "</li><li class=\"nav-item border-top pt-3 my-3\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if isPluginsPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "<a class=\"nav-link active\" href=\"#pluginsSubmenu\" data-bs-toggle=\"collapse\"><i class=\"fas fa-plug me-2\"></i>Plugins <i class=\"fas fa-chevron-down ms-auto\"></i></a> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, "<a class=\"nav-link collapsed\" href=\"#pluginsSubmenu\" data-bs-toggle=\"collapse\"><i class=\"fas fa-plug me-2\"></i>Plugins <i class=\"fas fa-chevron-down ms-auto\"></i></a> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
if isPluginsPage {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, "<div class=\"collapse show\" id=\"pluginsSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2 active\" href=\"/plugins\"><i class=\"fas fa-home me-2\"></i>Overview</a></li></ul></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
} else {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "<div class=\"collapse\" id=\"pluginsSubmenu\"><ul class=\"nav flex-column ms-3\"><li class=\"nav-item\"><a class=\"nav-link py-2\" href=\"/plugins\"><i class=\"fas fa-home me-2\"></i>Overview</a></li></ul></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "</li></ul></div></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-md-4\"><div class=\"pt-3\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 54, "</li></ul></div></div><!-- Main content --><main class=\"col-md-9 ms-sm-auto col-lg-10 px-md-4\"><div class=\"pt-3\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -524,43 +495,43 @@ func Layout(c *gin.Context, content templ.Component) templ.Component {
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "</div></main></div></div><!-- Footer --><footer class=\"footer mt-auto py-3 bg-light\"><div class=\"container-fluid text-center\"><small class=\"text-muted\">&copy; ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 55, "</div></main></div></div><!-- Footer --><footer class=\"footer mt-auto py-3 bg-light\"><div class=\"container-fluid text-center\"><small class=\"text-muted\">&copy; ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var26 string
templ_7745c5c3_Var26, templ_7745c5c3_Err = templ.JoinStringErrs(fmt.Sprintf("%d", time.Now().Year()))
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 387, Col: 60}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 350, Col: 60}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var26))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, " SeaweedFS Admin v")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 56, " SeaweedFS Admin v")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var27 string
templ_7745c5c3_Var27, templ_7745c5c3_Err = templ.JoinStringErrs(version.VERSION_NUMBER)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 387, Col: 102}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 350, Col: 102}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var27))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, " ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 57, " ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if !strings.Contains(version.VERSION, "enterprise") {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "<span class=\"mx-2\">•</span> <a href=\"https://seaweedfs.com\" target=\"_blank\" class=\"text-decoration-none\"><i class=\"fas fa-star me-1\"></i>Enterprise Version Available</a>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 58, "<span class=\"mx-2\">•</span> <a href=\"https://seaweedfs.com\" target=\"_blank\" class=\"text-decoration-none\"><i class=\"fas fa-star me-1\"></i>Enterprise Version Available</a>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "</small></div></footer><!-- Bootstrap JS --><script src=\"/static/js/bootstrap.bundle.min.js\"></script><!-- Modal Alerts JS (replaces native alert/confirm) --><script src=\"/static/js/modal-alerts.js\"></script><!-- Custom JS --><script src=\"/static/js/admin.js\"></script><script src=\"/static/js/iam-utils.js\"></script><script src=\"/static/js/s3tables.js\"></script></body></html>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 59, "</small></div></footer><!-- Bootstrap JS --><script src=\"/static/js/bootstrap.bundle.min.js\"></script><!-- Modal Alerts JS (replaces native alert/confirm) --><script src=\"/static/js/modal-alerts.js\"></script><!-- Custom JS --><script src=\"/static/js/admin.js\"></script><script src=\"/static/js/iam-utils.js\"></script><script src=\"/static/js/s3tables.js\"></script></body></html>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -589,56 +560,56 @@ func LoginForm(c *gin.Context, title string, errorMessage string) templ.Componen
templ_7745c5c3_Var28 = templ.NopComponent
}
ctx = templ.ClearChildren(ctx)
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 60, "<!doctype html><html lang=\"en\"><head><meta charset=\"UTF-8\"><title>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var29 string
templ_7745c5c3_Var29, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 415, Col: 17}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 378, Col: 17}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var29))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 66, " - Login</title><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"></head><body class=\"bg-light\"><div class=\"container\"><div class=\"row justify-content-center min-vh-100 align-items-center\"><div class=\"col-md-6 col-lg-4\"><div class=\"card shadow\"><div class=\"card-body p-5\"><div class=\"text-center mb-4\"><i class=\"fas fa-server fa-3x text-primary mb-3\"></i><h4 class=\"card-title\">")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 61, " - Login</title><link rel=\"icon\" href=\"/static/favicon.ico\" type=\"image/x-icon\"><meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><link href=\"/static/css/bootstrap.min.css\" rel=\"stylesheet\"><link href=\"/static/css/fontawesome.min.css\" rel=\"stylesheet\"></head><body class=\"bg-light\"><div class=\"container\"><div class=\"row justify-content-center min-vh-100 align-items-center\"><div class=\"col-md-6 col-lg-4\"><div class=\"card shadow\"><div class=\"card-body p-5\"><div class=\"text-center mb-4\"><i class=\"fas fa-server fa-3x text-primary mb-3\"></i><h4 class=\"card-title\">")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var30 string
templ_7745c5c3_Var30, templ_7745c5c3_Err = templ.JoinStringErrs(title)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 429, Col: 57}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 392, Col: 57}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var30))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 67, "</h4><p class=\"text-muted\">Please sign in to continue</p></div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 62, "</h4><p class=\"text-muted\">Please sign in to continue</p></div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
if errorMessage != "" {
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 68, "<div class=\"alert alert-danger\" role=\"alert\"><i class=\"fas fa-exclamation-triangle me-2\"></i> ")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 63, "<div class=\"alert alert-danger\" role=\"alert\"><i class=\"fas fa-exclamation-triangle me-2\"></i> ")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
var templ_7745c5c3_Var31 string
templ_7745c5c3_Var31, templ_7745c5c3_Err = templ.JoinStringErrs(errorMessage)
if templ_7745c5c3_Err != nil {
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 436, Col: 45}
return templ.Error{Err: templ_7745c5c3_Err, FileName: `weed/admin/view/layout/layout.templ`, Line: 399, Col: 45}
}
_, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var31))
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 69, "</div>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 64, "</div>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
}
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 70, "<form method=\"POST\" action=\"/login\"><div class=\"mb-3\"><label for=\"username\" class=\"form-label\">Username</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"fas fa-user\"></i></span> <input type=\"text\" class=\"form-control\" id=\"username\" name=\"username\" required></div></div><div class=\"mb-4\"><label for=\"password\" class=\"form-label\">Password</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"fas fa-lock\"></i></span> <input type=\"password\" class=\"form-control\" id=\"password\" name=\"password\" required></div></div><button type=\"submit\" class=\"btn btn-primary w-100\"><i class=\"fas fa-sign-in-alt me-2\"></i>Sign In</button></form></div></div></div></div></div><script src=\"/static/js/bootstrap.bundle.min.js\"></script></body></html>")
templ_7745c5c3_Err = templruntime.WriteString(templ_7745c5c3_Buffer, 65, "<form method=\"POST\" action=\"/login\"><div class=\"mb-3\"><label for=\"username\" class=\"form-label\">Username</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"fas fa-user\"></i></span> <input type=\"text\" class=\"form-control\" id=\"username\" name=\"username\" required></div></div><div class=\"mb-4\"><label for=\"password\" class=\"form-label\">Password</label><div class=\"input-group\"><span class=\"input-group-text\"><i class=\"fas fa-lock\"></i></span> <input type=\"password\" class=\"form-control\" id=\"password\" name=\"password\" required></div></div><button type=\"submit\" class=\"btn btn-primary w-100\"><i class=\"fas fa-sign-in-alt me-2\"></i>Sign In</button></form></div></div></div></div></div><script src=\"/static/js/bootstrap.bundle.min.js\"></script></body></html>")
if templ_7745c5c3_Err != nil {
return templ_7745c5c3_Err
}
@@ -1,14 +0,0 @@
package maintenance
const DefaultMasterMaintenanceScripts = `
lock
ec.encode -fullPercent=95 -quietFor=1h
ec.rebuild -apply
ec.balance -apply
fs.log.purge -daysAgo=7
volume.deleteEmpty -quietFor=24h -apply
volume.balance -apply
volume.fix.replication -apply
s3.clean.uploads -timeAgo=24h
unlock
`
+1 -1
View File
@@ -137,7 +137,7 @@ func backupFromLocation(volumeServer pb.ServerAddress, grpcDialOption grpc.DialO
// Handle compaction if needed
if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
if err = v.CompactByIndex(nil); err != nil {
if err = v.Compact2(0, 0, nil); err != nil {
v.Close()
return fmt.Errorf("compacting volume: %w", err), false
}
-1
View File
@@ -38,7 +38,6 @@ var Commands = []*Command{
cmdMqBroker,
cmdMqKafkaGateway,
cmdDB,
cmdPluginWorker,
cmdS3,
cmdScaffold,
cmdServer,
+9 -20
View File
@@ -1,8 +1,6 @@
package command
import (
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
@@ -20,9 +18,8 @@ var cmdCompact = &Command{
The compacted .dat file is stored as .cpd file.
The compacted .idx file is stored as .cpx file.
Supports two compaction methods:
* data: compacts based on the .dat file, works if .idx file is corrupted.
* index: compacts based on the .idx file, works if deletion happened but not written to .dat files.
For method=0, it compacts based on the .dat file, works if .idx file is corrupted.
For method=1, it compacts based on the .idx file, works if deletion happened but not written to .dat files.
`,
}
@@ -31,7 +28,7 @@ var (
compactVolumePath = cmdCompact.Flag.String("dir", ".", "data directory to store files")
compactVolumeCollection = cmdCompact.Flag.String("collection", "", "volume collection name")
compactVolumeId = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
compactMethod = cmdCompact.Flag.String("method", "data", "option to choose which compact method (data/index)")
compactMethod = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 (default) or 1.")
compactVolumePreallocate = cmdCompact.Flag.Int64("preallocateMB", 0, "preallocate volume disk space")
)
@@ -41,29 +38,21 @@ func runCompact(cmd *Command, args []string) bool {
return false
}
preallocateBytes := *compactVolumePreallocate * (1 << 20)
preallocate := *compactVolumePreallocate * (1 << 20)
vid := needle.VolumeId(*compactVolumeId)
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocateBytes, needle.GetCurrentVersion(), 0, 0)
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, needle.GetCurrentVersion(), 0, 0)
if err != nil {
glog.Fatalf("Load Volume [ERROR] %s\n", err)
}
opts := &storage.CompactOptions{
PreallocateBytes: preallocateBytes,
MaxBytesPerSecond: 0, // unlimited
}
switch strings.ToLower(*compactMethod) {
case "data":
if err = v.CompactByVolumeData(opts); err != nil {
if *compactMethod == 0 {
if err = v.Compact(preallocate, 0); err != nil {
glog.Fatalf("Compact Volume [ERROR] %s\n", err)
}
case "index":
if err = v.CompactByIndex(opts); err != nil {
} else {
if err = v.Compact2(preallocate, 0, nil); err != nil {
glog.Fatalf("Compact Volume [ERROR] %s\n", err)
}
default:
glog.Fatalf("unsupported compaction method %q", *compactMethod)
}
return true
+4 -14
View File
@@ -355,9 +355,6 @@ func isFlagPassed(name string) bool {
// isPortOpenOnIP checks if a port is available for binding on a specific IP address
func isPortOpenOnIP(ip string, port int) bool {
if port <= 0 || port > 65535 {
return false
}
listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", ip, port))
if err != nil {
return false
@@ -369,9 +366,6 @@ func isPortOpenOnIP(ip string, port int) bool {
// isPortAvailable checks if a port is available on any interface
// This is more comprehensive than checking a single IP
func isPortAvailable(port int) bool {
if port <= 0 || port > 65535 {
return false
}
// Try to listen on all interfaces (0.0.0.0)
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
@@ -387,10 +381,6 @@ func isPortAvailable(port int) bool {
func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPorts map[int]bool) int {
for i := 0; i < maxAttempts; i++ {
port := startPort + i
if port > 65535 {
// Wrap around to a lower range if we exceed 65535
port = 10000 + (port % 65535)
}
// Skip ports reserved for gRPC calculation
if reservedPorts[port] {
continue
@@ -408,15 +398,15 @@ func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPo
// If the port is not available, it finds the next available port and updates the pointer
// The reservedPorts map contains ports that should not be allocated (for gRPC collision avoidance)
func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reservedPorts map[int]bool, flagName string) error {
// Check if this port was explicitly specified by the user (from CLI, before config file was applied)
isExplicitPort := explicitPortFlags[flagName]
if *portPtr == 0 {
if portPtr == nil {
return nil
}
original := *portPtr
// Check if this port was explicitly specified by the user (from CLI, before config file was applied)
isExplicitPort := explicitPortFlags[flagName]
// Skip if this port is reserved for gRPC calculation
if reservedPorts[original] {
if isExplicitPort {
-321
View File
@@ -1,321 +0,0 @@
package command
import (
"context"
"fmt"
"os"
"os/signal"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/util/version"
"google.golang.org/grpc"
)
var cmdPluginWorker = &Command{
UsageLine: "plugin_worker -admin=<grpc_address> [-id=<worker_id>] [-plugins=<plugin_types>] [-workingDir=<path>]",
Short: "start a plugin-based worker using the new plugin system",
Long: `Start a worker using the new plugin system. This worker connects to the admin server
via gRPC and registers capabilities for handling maintenance tasks.
Supported plugins: erasure_coding, vacuum, balance
Examples:
weed plugin_worker -admin=localhost:23646
weed plugin_worker -admin=admin.example.com:23646 -id=worker-1
weed plugin_worker -admin=localhost:23646 -plugins=erasure_coding,vacuum
weed plugin_worker -admin=localhost:23646 -id=ec-worker-1 -workingDir=/tmp/worker
weed plugin_worker -admin=localhost:23646 -debug
`,
}
var (
pluginWorkerAdminServer = cmdPluginWorker.Flag.String("admin", "localhost:23646",
"admin server gRPC address (usually admin HTTP port + 10000)")
pluginWorkerID = cmdPluginWorker.Flag.String("id", "",
"plugin worker ID (auto-generated if not specified)")
pluginWorkerPlugins = cmdPluginWorker.Flag.String("plugins", "erasure_coding,vacuum,balance",
"comma-separated list of plugin types to enable")
pluginWorkerWorkingDir = cmdPluginWorker.Flag.String("workingDir", "",
"working directory for the plugin worker")
pluginWorkerMaxConcurrent = cmdPluginWorker.Flag.Int("maxConcurrent", 2,
"maximum number of concurrent jobs")
pluginWorkerDebug = cmdPluginWorker.Flag.Bool("debug", false,
"enable debug logging")
pluginWorkerDebugPort = cmdPluginWorker.Flag.Int("debug.port", 6061,
"http port for debugging")
pluginWorkerTimeout = cmdPluginWorker.Flag.Duration("timeout", 30*time.Second,
"gRPC connection timeout")
)
func init() {
cmdPluginWorker.Run = runPluginWorker
}
// GenericPluginWorker is a multi-plugin worker that connects to admin server
type GenericPluginWorker struct {
ID string
AdminServer pb.ServerAddress
Plugins []string
MaxConcurrentJobs int
WorkingDir string
PluginServiceClient plugin_pb.PluginServiceClient
Conn *grpc.ClientConn
Context context.Context
Cancel context.CancelFunc
mu sync.RWMutex
isRunning bool
}
// NewGenericPluginWorker creates a new generic plugin worker
func NewGenericPluginWorker(adminServer pb.ServerAddress, plugins []string, workingDir string, maxConcurrent int, id string) *GenericPluginWorker {
workerID := id
if workerID == "" {
workerID = fmt.Sprintf("worker-%s-%d", hostname(), time.Now().UnixNano())
}
return &GenericPluginWorker{
ID: workerID,
AdminServer: adminServer,
Plugins: plugins,
MaxConcurrentJobs: maxConcurrent,
WorkingDir: workingDir,
}
}
// Start connects to admin server and registers plugins
func (w *GenericPluginWorker) Start(ctx context.Context) error {
w.mu.Lock()
defer w.mu.Unlock()
if w.isRunning {
return fmt.Errorf("worker is already running")
}
w.Context, w.Cancel = context.WithCancel(ctx)
// Create gRPC connection
dialCtx, cancel := context.WithTimeout(w.Context, *pluginWorkerTimeout)
defer cancel()
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.worker")
conn, err := grpc.DialContext(dialCtx, w.AdminServer.ToGrpcAddress(), grpcDialOption)
if err != nil {
return fmt.Errorf("failed to connect to admin grpc server at %s: %v", w.AdminServer.ToGrpcAddress(), err)
}
w.Conn = conn
w.PluginServiceClient = plugin_pb.NewPluginServiceClient(conn)
glog.Infof("Successfully connected to admin grpc server at %s", w.AdminServer)
// Register plugin with admin server
capabilities := []string{}
for _, plugin := range w.Plugins {
plugin = strings.TrimSpace(plugin)
if plugin != "" {
capabilities = append(capabilities, plugin)
}
}
connectReq := &plugin_pb.PluginConnectRequest{
PluginId: w.ID,
PluginName: fmt.Sprintf("GenericWorker-%s", strings.Join(w.Plugins, ",")),
Version: version.VERSION,
Capabilities: capabilities,
MaxConcurrentJobs: int32(w.MaxConcurrentJobs),
SupportsStreaming: false,
Port: 0,
Metadata: map[string]string{
"working_dir": w.WorkingDir,
},
}
connectCtx, cancel := context.WithTimeout(w.Context, 10*time.Second)
defer cancel()
connectResp, err := w.PluginServiceClient.Connect(connectCtx, connectReq)
if err != nil {
return fmt.Errorf("failed to register with admin server: %v", err)
}
if !connectResp.Success {
return fmt.Errorf("plugin registration failed: %s", connectResp.Message)
}
glog.Infof("Plugin registered successfully. Assigned types: %v", connectResp.AssignedTypes)
w.isRunning = true
// Start background health reporting
go w.healthReportLoop()
glog.Infof("Plugin worker started successfully with ID: %s", w.ID)
return nil
}
// Stop gracefully stops the worker
func (w *GenericPluginWorker) Stop() error {
w.mu.Lock()
defer w.mu.Unlock()
if !w.isRunning {
return nil
}
w.isRunning = false
if w.Cancel != nil {
w.Cancel()
}
if w.Conn != nil {
return w.Conn.Close()
}
return nil
}
// healthReportLoop sends periodic health reports to admin server
func (w *GenericPluginWorker) healthReportLoop() {
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-w.Context.Done():
return
case <-ticker.C:
w.sendHealthReport()
}
}
}
// sendHealthReport sends a health report to the admin server
func (w *GenericPluginWorker) sendHealthReport() {
healthReport := &plugin_pb.HealthReport{
PluginId: w.ID,
TimestampMs: time.Now().UnixMilli(),
Status: plugin_pb.HealthStatus_HEALTH_STATUS_HEALTHY,
ActiveJobs: 0,
CpuPercent: 0,
MemoryBytes: 0,
JobProgress: []*plugin_pb.JobProgress{},
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := w.PluginServiceClient.ReportHealth(ctx, healthReport)
if err != nil {
glog.Warningf("Failed to send health report: %v", err)
}
}
// IsRunning checks if the worker is currently running
func (w *GenericPluginWorker) IsRunning() bool {
w.mu.RLock()
defer w.mu.RUnlock()
return w.isRunning
}
func hostname() string {
hostname, err := os.Hostname()
if err != nil {
hostname = "unknown"
}
return hostname
}
func runPluginWorker(cmd *Command, args []string) bool {
if *pluginWorkerDebug {
grace.StartDebugServer(*pluginWorkerDebugPort)
}
util.LoadConfiguration("security", false)
glog.Infof("Starting plugin worker (v%s)", version.VERSION)
glog.Infof("Admin server: %s", *pluginWorkerAdminServer)
if *pluginWorkerID != "" {
glog.Infof("Worker ID: %s", *pluginWorkerID)
}
glog.Infof("Plugins: %s", *pluginWorkerPlugins)
// Parse plugins
plugins := strings.Split(*pluginWorkerPlugins, ",")
validPlugins := []string{}
for _, p := range plugins {
p = strings.TrimSpace(p)
if p != "" {
validPlugins = append(validPlugins, p)
}
}
if len(validPlugins) == 0 {
glog.Fatalf("No valid plugins specified. Valid options: erasure_coding, vacuum, balance")
return false
}
// Set up working directory
workingDir := *pluginWorkerWorkingDir
if workingDir == "" {
var err error
workingDir, err = os.Getwd()
if err != nil {
glog.Fatalf("Failed to get current working directory: %v", err)
return false
}
}
// Create and validate working directory
if err := os.MkdirAll(workingDir, 0755); err != nil {
glog.Fatalf("Failed to create working directory: %v", err)
return false
}
glog.Infof("Working directory: %s", workingDir)
// Create plugin-specific subdirectories
for _, plugin := range validPlugins {
pluginDir := filepath.Join(workingDir, plugin)
if err := os.MkdirAll(pluginDir, 0755); err != nil {
glog.Fatalf("Failed to create plugin directory %s: %v", pluginDir, err)
return false
}
}
// Create and start the worker
adminServerAddress := pb.ServerAddress(*pluginWorkerAdminServer)
worker := NewGenericPluginWorker(adminServerAddress, validPlugins, workingDir, *pluginWorkerMaxConcurrent, *pluginWorkerID)
ctx := context.Background()
if err := worker.Start(ctx); err != nil {
glog.Fatalf("Failed to start plugin worker: %v", err)
return false
}
// Set up signal handling
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
glog.Infof("Plugin worker is running. Press Ctrl+C to stop")
// Wait for shutdown signal
<-sigChan
glog.Infof("Shutdown signal received, stopping plugin worker...")
if err := worker.Stop(); err != nil {
glog.Errorf("Error stopping plugin worker: %v", err)
}
glog.Infof("Plugin worker stopped")
return true
}
+1 -11
View File
@@ -1,15 +1,6 @@
package scaffold
import (
_ "embed"
"strings"
"github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
)
func init() {
Master = strings.ReplaceAll(masterTemplate, "{{DEFAULT_MAINTENANCE_SCRIPTS}}", maintenance.DefaultMasterMaintenanceScripts)
}
import _ "embed"
//go:embed filer.toml
var Filer string
@@ -24,7 +15,6 @@ var Replication string
var Security string
//go:embed master.toml
var masterTemplate string
var Master string
//go:embed shell.toml
+11 -1
View File
@@ -6,7 +6,17 @@
[master.maintenance]
# periodically run these scripts are the same as running them from 'weed shell'
scripts = """{{DEFAULT_MAINTENANCE_SCRIPTS}}"""
scripts = """
lock
ec.encode -fullPercent=95 -quietFor=1h
ec.rebuild -apply
ec.balance -apply
volume.deleteEmpty -quietFor=24h -apply
volume.balance -apply
volume.fix.replication -apply
s3.clean.uploads -timeAgo=24h
unlock
"""
sleep_minutes = 17 # sleep minutes between each script execution
-1
View File
@@ -43,7 +43,6 @@ expires_after_seconds = 10 # seconds
# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
# - f.e. the S3 API Shim generates the JWT
# - the Filer server validates the JWT on writing
# NOTE: This key is ALSO used as a fallback signing key for S3 STS if s3.iam.config does not specify a signingKey.
# the jwt defaults to expire after 10 seconds.
[jwt.filer_signing]
key = ""
-8
View File
@@ -654,14 +654,6 @@ func (m *IAMManager) GetSTSService() *sts.STSService {
return m.stsService
}
// DefaultAllow returns whether the default effect is Allow
func (m *IAMManager) DefaultAllow() bool {
if !m.initialized || m.policyEngine == nil {
return true // Default to true if not initialized
}
return m.policyEngine.DefaultAllow()
}
// parseJWTTokenForTrustPolicy parses a JWT token to extract claims for trust policy evaluation
func parseJWTTokenForTrustPolicy(tokenString string) (map[string]interface{}, error) {
// Simple JWT parsing without verification (for trust policy context only)
-8
View File
@@ -324,14 +324,6 @@ func (e *PolicyEngine) IsInitialized() bool {
return e.initialized
}
// DefaultAllow returns whether the default effect is Allow
func (e *PolicyEngine) DefaultAllow() bool {
if e.config == nil {
return true // Default to Allow if not configured
}
return e.config.DefaultEffect == string(EffectAllow)
}
// AddPolicy adds a policy to the engine (filerAddress ignored for memory stores)
func (e *PolicyEngine) AddPolicy(filerAddress string, name string, policy *PolicyDocument) error {
if !e.initialized {
-18
View File
@@ -270,9 +270,6 @@ func (s *STSService) Initialize(config *STSConfig) error {
return fmt.Errorf(ErrConfigCannotBeNil)
}
// Apply defaults before validation
config.ApplyDefaults()
if err := s.validateConfig(config); err != nil {
return fmt.Errorf("invalid STS configuration: %w", err)
}
@@ -291,21 +288,6 @@ func (s *STSService) Initialize(config *STSConfig) error {
return nil
}
// ApplyDefaults applies default values to the STS configuration
func (c *STSConfig) ApplyDefaults() {
if c.TokenDuration.Duration <= 0 {
c.TokenDuration.Duration = time.Duration(DefaultTokenDuration) * time.Second
}
if c.MaxSessionLength.Duration <= 0 {
c.MaxSessionLength.Duration = time.Duration(DefaultMaxSessionLength) * time.Second
}
if c.Issuer == "" {
c.Issuer = DefaultIssuer
}
}
// validateConfig validates the STS configuration
func (s *STSService) validateConfig(config *STSConfig) error {
if config.TokenDuration.Duration <= 0 {
-23
View File
@@ -75,34 +75,11 @@ func TestSTSServiceInitialization(t *testing.T) {
} else {
assert.NoError(t, err)
assert.True(t, service.IsInitialized())
// Verify defaults if applicable
if tt.config.Issuer == "" {
assert.Equal(t, DefaultIssuer, service.Config.Issuer)
}
if tt.config.TokenDuration.Duration == 0 {
assert.Equal(t, time.Duration(DefaultTokenDuration)*time.Second, service.Config.TokenDuration.Duration)
}
}
})
}
}
func TestSTSServiceDefaults(t *testing.T) {
service := NewSTSService()
config := &STSConfig{
SigningKey: []byte("test-signing-key"),
// Missing duration and issuer
}
err := service.Initialize(config)
assert.NoError(t, err)
assert.Equal(t, DefaultIssuer, config.Issuer)
assert.Equal(t, time.Duration(DefaultTokenDuration)*time.Second, config.TokenDuration.Duration)
assert.Equal(t, time.Duration(DefaultMaxSessionLength)*time.Second, config.MaxSessionLength.Duration)
}
// TestAssumeRoleWithWebIdentity tests role assumption with OIDC tokens
func TestAssumeRoleWithWebIdentity(t *testing.T) {
service := setupTestSTSService(t)
+1 -10
View File
@@ -89,9 +89,7 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli
GrpcDialOption: option.GrpcDialOption,
}
// Initialize FilerClient for IAM - explicit filers only (no discovery as FilerGroup unspecified)
filerClient := wdclient.NewFilerClient(option.Filers, option.GrpcDialOption, "")
iam := s3api.NewIdentityAccessManagementWithStore(&s3Option, filerClient, explicitStore)
iam := s3api.NewIdentityAccessManagementWithStore(&s3Option, explicitStore)
configure.credentialManager = iam.GetCredentialManager()
iamApiServer = &IamApiServer{
@@ -102,13 +100,6 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli
masterClient: masterClient,
}
// Keep attempting to load configuration from filer now that we have a client
go func() {
if err := iam.LoadS3ApiConfigurationFromCredentialManager(); err != nil {
glog.Warningf("Failed to load IAM config from credential manager after client update: %v", err)
}
}()
iamApiServer.registerRouter(router)
return iamApiServer, nil
-1
View File
@@ -14,7 +14,6 @@ gen:
protoc mq_schema.proto --go_out=./schema_pb --go-grpc_out=./schema_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc mq_agent.proto --go_out=./mq_agent_pb --go-grpc_out=./mq_agent_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc worker.proto --go_out=./worker_pb --go-grpc_out=./worker_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
protoc plugin.proto --go_out=./plugin_pb --go-grpc_out=./plugin_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
# protoc filer.proto --java_out=../../other/java/client/src/main/java
cp filer.proto ../../other/java/client/src/main/proto
-445
View File
@@ -1,445 +0,0 @@
syntax = "proto3";
package plugin;
option go_package = "github.com/seaweedfs/seaweedfs/weed/pb/plugin_pb";
option java_package = "seaweedfs.plugin";
option java_outer_classname = "PluginProto";
import "google/protobuf/timestamp.proto";
import "google/protobuf/duration.proto";
// PluginService is the core service for plugin lifecycle and job execution
service PluginService {
// Connect registers a plugin with the master
rpc Connect(PluginConnectRequest) returns (PluginConnectResponse);
// ExecuteJob processes a detection or maintenance job
rpc ExecuteJob(ExecuteJobRequest) returns (ExecuteJobResponse);
// ReportHealth sends periodic health status updates
rpc ReportHealth(HealthReport) returns (HealthReportResponse);
// GetConfig retrieves the latest configuration
rpc GetConfig(GetConfigRequest) returns (GetConfigResponse);
// SubmitResult sends job execution results back to master
rpc SubmitResult(JobResultRequest) returns (JobResultResponse);
}
// AdminQueryService provides monitoring and diagnostics endpoints
service AdminQueryService {
// GetPluginStats returns statistics for all connected plugins
rpc GetPluginStats(GetPluginStatsRequest) returns (GetPluginStatsResponse);
// ListPlugins returns information about all registered plugins
rpc ListPlugins(ListPluginsRequest) returns (ListPluginsResponse);
// ListJobs returns current and historical job information
rpc ListJobs(ListJobsRequest) returns (ListJobsResponse);
// GetJobStatus returns detailed status of a specific job
rpc GetJobStatus(GetJobStatusRequest) returns (GetJobStatusResponse);
// GetPluginLogs returns logs from a specific plugin
rpc GetPluginLogs(GetPluginLogsRequest) returns (GetPluginLogsResponse);
}
// AdminCommandService provides administrative operations
service AdminCommandService {
// SaveConfig persists plugin configuration
rpc SaveConfig(SaveConfigRequest) returns (SaveConfigResponse);
// ReloadConfig reloads configuration without restarting
rpc ReloadConfig(ReloadConfigRequest) returns (ReloadConfigResponse);
// EnablePlugin enables a specific plugin
rpc EnablePlugin(EnablePluginRequest) returns (EnablePluginResponse);
// DisablePlugin disables a specific plugin
rpc DisablePlugin(DisablePluginRequest) returns (DisablePluginResponse);
// TriggerDetection manually triggers a detection for specific types
rpc TriggerDetection(TriggerDetectionRequest) returns (TriggerDetectionResponse);
// CancelJob cancels a running job
rpc CancelJob(CancelJobRequest) returns (CancelJobResponse);
// PurgeHistory clears job history
rpc PurgeHistory(PurgeHistoryRequest) returns (PurgeHistoryResponse);
}
// ============================================================================
// Core Messages
// ============================================================================
message PluginConnectRequest {
string plugin_id = 1;
string plugin_name = 2;
string version = 3;
repeated string capabilities = 4;
PluginCapabilities capabilities_detail = 5;
int32 max_concurrent_jobs = 6;
bool supports_streaming = 7;
int32 port = 8;
map<string, string> metadata = 9;
}
message PluginConnectResponse {
bool success = 1;
string message = 2;
string master_id = 3;
repeated string assigned_types = 4;
PluginConfig config = 5;
}
message PluginCapabilities {
repeated DetectionCapability detection = 1;
repeated MaintenanceCapability maintenance = 2;
repeated string supported_datasources = 3;
}
message DetectionCapability {
string type = 1;
string description = 2;
int32 min_interval_seconds = 3;
bool requires_full_scan = 4;
repeated string output_metrics = 5;
}
message MaintenanceCapability {
string type = 1;
string description = 2;
repeated string required_detection_types = 3;
int32 estimated_duration_seconds = 4;
}
message ExecuteJobRequest {
string job_id = 1;
string job_type = 2;
JobPayload payload = 3;
google.protobuf.Duration timeout = 4;
int32 retry_count = 5;
map<string, string> context = 6;
}
message ExecuteJobResponse {
string job_id = 1;
ExecutionStatus status = 2;
string message = 3;
}
enum ExecutionStatus {
EXECUTION_STATUS_UNKNOWN = 0;
EXECUTION_STATUS_ACCEPTED = 1;
EXECUTION_STATUS_RUNNING = 2;
EXECUTION_STATUS_COMPLETED = 3;
EXECUTION_STATUS_FAILED = 4;
EXECUTION_STATUS_CANCELLED = 5;
}
message JobPayload {
string detection_type = 1;
string target_datasource = 2;
bytes data = 3;
map<string, string> parameters = 4;
int64 timestamp_ms = 5;
}
message HealthReport {
string plugin_id = 1;
int64 timestamp_ms = 2;
HealthStatus status = 3;
int32 active_jobs = 4;
int64 cpu_percent = 5;
int64 memory_bytes = 6;
repeated JobProgress job_progress = 7;
}
enum HealthStatus {
HEALTH_STATUS_HEALTHY = 0;
HEALTH_STATUS_DEGRADED = 1;
HEALTH_STATUS_UNHEALTHY = 2;
}
message JobProgress {
string job_id = 1;
float progress_percent = 2;
string current_step = 3;
}
message HealthReportResponse {
bool acknowledged = 1;
string feedback = 2;
}
message GetConfigRequest {
string plugin_id = 1;
bool include_defaults = 2;
}
message GetConfigResponse {
PluginConfig config = 1;
int64 version = 2;
}
message PluginConfig {
string plugin_id = 1;
map<string, string> properties = 2;
repeated JobTypeConfig job_types = 3;
int32 max_retries = 4;
google.protobuf.Duration health_check_interval = 5;
google.protobuf.Duration job_timeout = 6;
map<string, string> environment = 7;
}
message JobTypeConfig {
string type = 1;
bool enabled = 2;
int32 priority = 3;
google.protobuf.Duration interval = 4;
int32 max_concurrent = 5;
map<string, string> parameters = 6;
}
message JobResultRequest {
string job_id = 1;
string job_type = 2;
ExecutionStatus status = 3;
string message = 4;
JobResult result = 5;
google.protobuf.Duration execution_time = 6;
int32 retry_count_used = 7;
}
message JobResult {
bool success = 1;
bytes data = 2;
repeated DetectionRecord detections = 3;
repeated string warnings = 4;
repeated string errors = 5;
map<string, string> metadata = 6;
}
message DetectionRecord {
string detection_type = 1;
int64 timestamp_ms = 2;
string severity = 3;
string description = 4;
string affected_resource = 5;
bytes raw_data = 6;
}
message JobResultResponse {
bool acknowledged = 1;
repeated string actions_to_take = 2;
}
// ============================================================================
// Query Messages
// ============================================================================
message GetPluginStatsRequest {
string plugin_id = 1;
}
message GetPluginStatsResponse {
repeated PluginStats stats = 1;
}
message PluginStats {
string plugin_id = 1;
string status = 2;
int32 active_jobs = 3;
int32 completed_jobs = 4;
int32 failed_jobs = 5;
int64 total_detections = 6;
float avg_execution_time_ms = 7;
float cpu_usage_percent = 8;
int64 memory_usage_bytes = 9;
google.protobuf.Timestamp last_heartbeat = 10;
int32 uptime_seconds = 11;
}
message ListPluginsRequest {
bool include_disabled = 1;
repeated string filter_by_capability = 2;
}
message ListPluginsResponse {
repeated PluginInfo plugins = 1;
}
message PluginInfo {
string plugin_id = 1;
string name = 2;
string version = 3;
string status = 4;
repeated string capabilities = 5;
int32 max_concurrent_jobs = 6;
int32 active_jobs = 7;
google.protobuf.Timestamp connected_at = 8;
google.protobuf.Timestamp last_heartbeat = 9;
map<string, string> metadata = 10;
}
message ListJobsRequest {
string plugin_id = 1;
JobState filter_state = 2;
int32 limit = 3;
int32 offset = 4;
bool include_history = 5;
}
message ListJobsResponse {
repeated JobInfo jobs = 1;
int32 total_count = 2;
}
message JobInfo {
string job_id = 1;
string job_type = 2;
string plugin_id = 3;
JobState state = 4;
google.protobuf.Timestamp created_at = 5;
google.protobuf.Timestamp started_at = 6;
google.protobuf.Timestamp completed_at = 7;
google.protobuf.Duration execution_time = 8;
int32 retry_count = 9;
string last_error = 10;
}
enum JobState {
JOB_STATE_PENDING = 0;
JOB_STATE_SCHEDULED = 1;
JOB_STATE_RUNNING = 2;
JOB_STATE_COMPLETED = 3;
JOB_STATE_FAILED = 4;
JOB_STATE_CANCELLED = 5;
}
message GetJobStatusRequest {
string job_id = 1;
}
message GetJobStatusResponse {
JobInfo job_info = 1;
JobResult result = 2;
string detailed_status = 3;
}
message GetPluginLogsRequest {
string plugin_id = 1;
int32 lines = 2;
int64 since_timestamp_ms = 3;
string log_level = 4;
}
message GetPluginLogsResponse {
repeated LogEntry entries = 1;
}
message LogEntry {
int64 timestamp_ms = 1;
string level = 2;
string message = 3;
map<string, string> context = 4;
}
// ============================================================================
// Command Messages
// ============================================================================
message SaveConfigRequest {
PluginConfig config = 1;
bool backup_existing = 2;
}
message SaveConfigResponse {
bool success = 1;
string message = 2;
int64 config_version = 3;
}
message ReloadConfigRequest {
string plugin_id = 1;
}
message ReloadConfigResponse {
bool success = 1;
string message = 2;
}
message EnablePluginRequest {
string plugin_id = 1;
}
message EnablePluginResponse {
bool success = 1;
string message = 2;
}
message DisablePluginRequest {
string plugin_id = 1;
}
message DisablePluginResponse {
bool success = 1;
string message = 2;
}
message TriggerDetectionRequest {
repeated string detection_types = 1;
string target_datasource = 2;
int32 parallelism = 3;
}
message TriggerDetectionResponse {
bool success = 1;
repeated string triggered_job_ids = 2;
}
message CancelJobRequest {
string job_id = 1;
bool force = 2;
}
message CancelJobResponse {
bool success = 1;
string message = 2;
}
message PurgeHistoryRequest {
int64 before_timestamp_ms = 1;
repeated JobState states_to_purge = 2;
}
message PurgeHistoryResponse {
bool success = 1;
int32 records_deleted = 2;
}
// ============================================================================
// Execution Records (for persistence)
// ============================================================================
message ExecutionRecord {
string job_id = 1;
string job_type = 2;
string plugin_id = 3;
JobState state = 4;
google.protobuf.Timestamp created_at = 5;
google.protobuf.Timestamp started_at = 6;
google.protobuf.Timestamp completed_at = 7;
JobPayload payload = 8;
JobResult result = 9;
int32 retry_count = 10;
string last_error = 11;
}
message ConfigSnapshot {
string plugin_id = 1;
PluginConfig config = 2;
google.protobuf.Timestamp created_at = 3;
int64 version = 4;
}
File diff suppressed because it is too large Load Diff
-903
View File
@@ -1,903 +0,0 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
// - protoc-gen-go-grpc v1.5.1
// - protoc v6.33.4
// source: plugin.proto
package plugin_pb
import (
context "context"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
)
// This is a compile-time assertion to ensure that this generated file
// is compatible with the grpc package it is being compiled against.
// Requires gRPC-Go v1.64.0 or later.
const _ = grpc.SupportPackageIsVersion9
const (
PluginService_Connect_FullMethodName = "/plugin.PluginService/Connect"
PluginService_ExecuteJob_FullMethodName = "/plugin.PluginService/ExecuteJob"
PluginService_ReportHealth_FullMethodName = "/plugin.PluginService/ReportHealth"
PluginService_GetConfig_FullMethodName = "/plugin.PluginService/GetConfig"
PluginService_SubmitResult_FullMethodName = "/plugin.PluginService/SubmitResult"
)
// PluginServiceClient is the client API for PluginService service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
//
// PluginService is the core service for plugin lifecycle and job execution
type PluginServiceClient interface {
// Connect registers a plugin with the master
Connect(ctx context.Context, in *PluginConnectRequest, opts ...grpc.CallOption) (*PluginConnectResponse, error)
// ExecuteJob processes a detection or maintenance job
ExecuteJob(ctx context.Context, in *ExecuteJobRequest, opts ...grpc.CallOption) (*ExecuteJobResponse, error)
// ReportHealth sends periodic health status updates
ReportHealth(ctx context.Context, in *HealthReport, opts ...grpc.CallOption) (*HealthReportResponse, error)
// GetConfig retrieves the latest configuration
GetConfig(ctx context.Context, in *GetConfigRequest, opts ...grpc.CallOption) (*GetConfigResponse, error)
// SubmitResult sends job execution results back to master
SubmitResult(ctx context.Context, in *JobResultRequest, opts ...grpc.CallOption) (*JobResultResponse, error)
}
type pluginServiceClient struct {
cc grpc.ClientConnInterface
}
func NewPluginServiceClient(cc grpc.ClientConnInterface) PluginServiceClient {
return &pluginServiceClient{cc}
}
func (c *pluginServiceClient) Connect(ctx context.Context, in *PluginConnectRequest, opts ...grpc.CallOption) (*PluginConnectResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(PluginConnectResponse)
err := c.cc.Invoke(ctx, PluginService_Connect_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *pluginServiceClient) ExecuteJob(ctx context.Context, in *ExecuteJobRequest, opts ...grpc.CallOption) (*ExecuteJobResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ExecuteJobResponse)
err := c.cc.Invoke(ctx, PluginService_ExecuteJob_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *pluginServiceClient) ReportHealth(ctx context.Context, in *HealthReport, opts ...grpc.CallOption) (*HealthReportResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(HealthReportResponse)
err := c.cc.Invoke(ctx, PluginService_ReportHealth_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *pluginServiceClient) GetConfig(ctx context.Context, in *GetConfigRequest, opts ...grpc.CallOption) (*GetConfigResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(GetConfigResponse)
err := c.cc.Invoke(ctx, PluginService_GetConfig_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *pluginServiceClient) SubmitResult(ctx context.Context, in *JobResultRequest, opts ...grpc.CallOption) (*JobResultResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(JobResultResponse)
err := c.cc.Invoke(ctx, PluginService_SubmitResult_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// PluginServiceServer is the server API for PluginService service.
// All implementations must embed UnimplementedPluginServiceServer
// for forward compatibility.
//
// PluginService is the core service for plugin lifecycle and job execution
type PluginServiceServer interface {
// Connect registers a plugin with the master
Connect(context.Context, *PluginConnectRequest) (*PluginConnectResponse, error)
// ExecuteJob processes a detection or maintenance job
ExecuteJob(context.Context, *ExecuteJobRequest) (*ExecuteJobResponse, error)
// ReportHealth sends periodic health status updates
ReportHealth(context.Context, *HealthReport) (*HealthReportResponse, error)
// GetConfig retrieves the latest configuration
GetConfig(context.Context, *GetConfigRequest) (*GetConfigResponse, error)
// SubmitResult sends job execution results back to master
SubmitResult(context.Context, *JobResultRequest) (*JobResultResponse, error)
mustEmbedUnimplementedPluginServiceServer()
}
// UnimplementedPluginServiceServer must be embedded to have
// forward compatible implementations.
//
// NOTE: this should be embedded by value instead of pointer to avoid a nil
// pointer dereference when methods are called.
type UnimplementedPluginServiceServer struct{}
func (UnimplementedPluginServiceServer) Connect(context.Context, *PluginConnectRequest) (*PluginConnectResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method Connect not implemented")
}
func (UnimplementedPluginServiceServer) ExecuteJob(context.Context, *ExecuteJobRequest) (*ExecuteJobResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ExecuteJob not implemented")
}
func (UnimplementedPluginServiceServer) ReportHealth(context.Context, *HealthReport) (*HealthReportResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReportHealth not implemented")
}
func (UnimplementedPluginServiceServer) GetConfig(context.Context, *GetConfigRequest) (*GetConfigResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetConfig not implemented")
}
func (UnimplementedPluginServiceServer) SubmitResult(context.Context, *JobResultRequest) (*JobResultResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method SubmitResult not implemented")
}
func (UnimplementedPluginServiceServer) mustEmbedUnimplementedPluginServiceServer() {}
func (UnimplementedPluginServiceServer) testEmbeddedByValue() {}
// UnsafePluginServiceServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to PluginServiceServer will
// result in compilation errors.
type UnsafePluginServiceServer interface {
mustEmbedUnimplementedPluginServiceServer()
}
func RegisterPluginServiceServer(s grpc.ServiceRegistrar, srv PluginServiceServer) {
// If the following call pancis, it indicates UnimplementedPluginServiceServer was
// embedded by pointer and is nil. This will cause panics if an
// unimplemented method is ever invoked, so we test this at initialization
// time to prevent it from happening at runtime later due to I/O.
if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
t.testEmbeddedByValue()
}
s.RegisterService(&PluginService_ServiceDesc, srv)
}
func _PluginService_Connect_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(PluginConnectRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(PluginServiceServer).Connect(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: PluginService_Connect_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PluginServiceServer).Connect(ctx, req.(*PluginConnectRequest))
}
return interceptor(ctx, in, info, handler)
}
func _PluginService_ExecuteJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ExecuteJobRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(PluginServiceServer).ExecuteJob(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: PluginService_ExecuteJob_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PluginServiceServer).ExecuteJob(ctx, req.(*ExecuteJobRequest))
}
return interceptor(ctx, in, info, handler)
}
func _PluginService_ReportHealth_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(HealthReport)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(PluginServiceServer).ReportHealth(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: PluginService_ReportHealth_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PluginServiceServer).ReportHealth(ctx, req.(*HealthReport))
}
return interceptor(ctx, in, info, handler)
}
func _PluginService_GetConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetConfigRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(PluginServiceServer).GetConfig(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: PluginService_GetConfig_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PluginServiceServer).GetConfig(ctx, req.(*GetConfigRequest))
}
return interceptor(ctx, in, info, handler)
}
func _PluginService_SubmitResult_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(JobResultRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(PluginServiceServer).SubmitResult(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: PluginService_SubmitResult_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(PluginServiceServer).SubmitResult(ctx, req.(*JobResultRequest))
}
return interceptor(ctx, in, info, handler)
}
// PluginService_ServiceDesc is the grpc.ServiceDesc for PluginService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var PluginService_ServiceDesc = grpc.ServiceDesc{
ServiceName: "plugin.PluginService",
HandlerType: (*PluginServiceServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "Connect",
Handler: _PluginService_Connect_Handler,
},
{
MethodName: "ExecuteJob",
Handler: _PluginService_ExecuteJob_Handler,
},
{
MethodName: "ReportHealth",
Handler: _PluginService_ReportHealth_Handler,
},
{
MethodName: "GetConfig",
Handler: _PluginService_GetConfig_Handler,
},
{
MethodName: "SubmitResult",
Handler: _PluginService_SubmitResult_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "plugin.proto",
}
const (
AdminQueryService_GetPluginStats_FullMethodName = "/plugin.AdminQueryService/GetPluginStats"
AdminQueryService_ListPlugins_FullMethodName = "/plugin.AdminQueryService/ListPlugins"
AdminQueryService_ListJobs_FullMethodName = "/plugin.AdminQueryService/ListJobs"
AdminQueryService_GetJobStatus_FullMethodName = "/plugin.AdminQueryService/GetJobStatus"
AdminQueryService_GetPluginLogs_FullMethodName = "/plugin.AdminQueryService/GetPluginLogs"
)
// AdminQueryServiceClient is the client API for AdminQueryService service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
//
// AdminQueryService provides monitoring and diagnostics endpoints
type AdminQueryServiceClient interface {
// GetPluginStats returns statistics for all connected plugins
GetPluginStats(ctx context.Context, in *GetPluginStatsRequest, opts ...grpc.CallOption) (*GetPluginStatsResponse, error)
// ListPlugins returns information about all registered plugins
ListPlugins(ctx context.Context, in *ListPluginsRequest, opts ...grpc.CallOption) (*ListPluginsResponse, error)
// ListJobs returns current and historical job information
ListJobs(ctx context.Context, in *ListJobsRequest, opts ...grpc.CallOption) (*ListJobsResponse, error)
// GetJobStatus returns detailed status of a specific job
GetJobStatus(ctx context.Context, in *GetJobStatusRequest, opts ...grpc.CallOption) (*GetJobStatusResponse, error)
// GetPluginLogs returns logs from a specific plugin
GetPluginLogs(ctx context.Context, in *GetPluginLogsRequest, opts ...grpc.CallOption) (*GetPluginLogsResponse, error)
}
type adminQueryServiceClient struct {
cc grpc.ClientConnInterface
}
func NewAdminQueryServiceClient(cc grpc.ClientConnInterface) AdminQueryServiceClient {
return &adminQueryServiceClient{cc}
}
func (c *adminQueryServiceClient) GetPluginStats(ctx context.Context, in *GetPluginStatsRequest, opts ...grpc.CallOption) (*GetPluginStatsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(GetPluginStatsResponse)
err := c.cc.Invoke(ctx, AdminQueryService_GetPluginStats_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *adminQueryServiceClient) ListPlugins(ctx context.Context, in *ListPluginsRequest, opts ...grpc.CallOption) (*ListPluginsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ListPluginsResponse)
err := c.cc.Invoke(ctx, AdminQueryService_ListPlugins_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *adminQueryServiceClient) ListJobs(ctx context.Context, in *ListJobsRequest, opts ...grpc.CallOption) (*ListJobsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ListJobsResponse)
err := c.cc.Invoke(ctx, AdminQueryService_ListJobs_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *adminQueryServiceClient) GetJobStatus(ctx context.Context, in *GetJobStatusRequest, opts ...grpc.CallOption) (*GetJobStatusResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(GetJobStatusResponse)
err := c.cc.Invoke(ctx, AdminQueryService_GetJobStatus_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *adminQueryServiceClient) GetPluginLogs(ctx context.Context, in *GetPluginLogsRequest, opts ...grpc.CallOption) (*GetPluginLogsResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(GetPluginLogsResponse)
err := c.cc.Invoke(ctx, AdminQueryService_GetPluginLogs_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// AdminQueryServiceServer is the server API for AdminQueryService service.
// All implementations must embed UnimplementedAdminQueryServiceServer
// for forward compatibility.
//
// AdminQueryService provides monitoring and diagnostics endpoints
type AdminQueryServiceServer interface {
// GetPluginStats returns statistics for all connected plugins
GetPluginStats(context.Context, *GetPluginStatsRequest) (*GetPluginStatsResponse, error)
// ListPlugins returns information about all registered plugins
ListPlugins(context.Context, *ListPluginsRequest) (*ListPluginsResponse, error)
// ListJobs returns current and historical job information
ListJobs(context.Context, *ListJobsRequest) (*ListJobsResponse, error)
// GetJobStatus returns detailed status of a specific job
GetJobStatus(context.Context, *GetJobStatusRequest) (*GetJobStatusResponse, error)
// GetPluginLogs returns logs from a specific plugin
GetPluginLogs(context.Context, *GetPluginLogsRequest) (*GetPluginLogsResponse, error)
mustEmbedUnimplementedAdminQueryServiceServer()
}
// UnimplementedAdminQueryServiceServer must be embedded to have
// forward compatible implementations.
//
// NOTE: this should be embedded by value instead of pointer to avoid a nil
// pointer dereference when methods are called.
type UnimplementedAdminQueryServiceServer struct{}
func (UnimplementedAdminQueryServiceServer) GetPluginStats(context.Context, *GetPluginStatsRequest) (*GetPluginStatsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetPluginStats not implemented")
}
func (UnimplementedAdminQueryServiceServer) ListPlugins(context.Context, *ListPluginsRequest) (*ListPluginsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListPlugins not implemented")
}
func (UnimplementedAdminQueryServiceServer) ListJobs(context.Context, *ListJobsRequest) (*ListJobsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ListJobs not implemented")
}
func (UnimplementedAdminQueryServiceServer) GetJobStatus(context.Context, *GetJobStatusRequest) (*GetJobStatusResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetJobStatus not implemented")
}
func (UnimplementedAdminQueryServiceServer) GetPluginLogs(context.Context, *GetPluginLogsRequest) (*GetPluginLogsResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetPluginLogs not implemented")
}
func (UnimplementedAdminQueryServiceServer) mustEmbedUnimplementedAdminQueryServiceServer() {}
func (UnimplementedAdminQueryServiceServer) testEmbeddedByValue() {}
// UnsafeAdminQueryServiceServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to AdminQueryServiceServer will
// result in compilation errors.
type UnsafeAdminQueryServiceServer interface {
mustEmbedUnimplementedAdminQueryServiceServer()
}
func RegisterAdminQueryServiceServer(s grpc.ServiceRegistrar, srv AdminQueryServiceServer) {
// If the following call pancis, it indicates UnimplementedAdminQueryServiceServer was
// embedded by pointer and is nil. This will cause panics if an
// unimplemented method is ever invoked, so we test this at initialization
// time to prevent it from happening at runtime later due to I/O.
if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
t.testEmbeddedByValue()
}
s.RegisterService(&AdminQueryService_ServiceDesc, srv)
}
func _AdminQueryService_GetPluginStats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetPluginStatsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminQueryServiceServer).GetPluginStats(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminQueryService_GetPluginStats_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminQueryServiceServer).GetPluginStats(ctx, req.(*GetPluginStatsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AdminQueryService_ListPlugins_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListPluginsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminQueryServiceServer).ListPlugins(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminQueryService_ListPlugins_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminQueryServiceServer).ListPlugins(ctx, req.(*ListPluginsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AdminQueryService_ListJobs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ListJobsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminQueryServiceServer).ListJobs(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminQueryService_ListJobs_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminQueryServiceServer).ListJobs(ctx, req.(*ListJobsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AdminQueryService_GetJobStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetJobStatusRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminQueryServiceServer).GetJobStatus(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminQueryService_GetJobStatus_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminQueryServiceServer).GetJobStatus(ctx, req.(*GetJobStatusRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AdminQueryService_GetPluginLogs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(GetPluginLogsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminQueryServiceServer).GetPluginLogs(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminQueryService_GetPluginLogs_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminQueryServiceServer).GetPluginLogs(ctx, req.(*GetPluginLogsRequest))
}
return interceptor(ctx, in, info, handler)
}
// AdminQueryService_ServiceDesc is the grpc.ServiceDesc for AdminQueryService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var AdminQueryService_ServiceDesc = grpc.ServiceDesc{
ServiceName: "plugin.AdminQueryService",
HandlerType: (*AdminQueryServiceServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "GetPluginStats",
Handler: _AdminQueryService_GetPluginStats_Handler,
},
{
MethodName: "ListPlugins",
Handler: _AdminQueryService_ListPlugins_Handler,
},
{
MethodName: "ListJobs",
Handler: _AdminQueryService_ListJobs_Handler,
},
{
MethodName: "GetJobStatus",
Handler: _AdminQueryService_GetJobStatus_Handler,
},
{
MethodName: "GetPluginLogs",
Handler: _AdminQueryService_GetPluginLogs_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "plugin.proto",
}
const (
AdminCommandService_SaveConfig_FullMethodName = "/plugin.AdminCommandService/SaveConfig"
AdminCommandService_ReloadConfig_FullMethodName = "/plugin.AdminCommandService/ReloadConfig"
AdminCommandService_EnablePlugin_FullMethodName = "/plugin.AdminCommandService/EnablePlugin"
AdminCommandService_DisablePlugin_FullMethodName = "/plugin.AdminCommandService/DisablePlugin"
AdminCommandService_TriggerDetection_FullMethodName = "/plugin.AdminCommandService/TriggerDetection"
AdminCommandService_CancelJob_FullMethodName = "/plugin.AdminCommandService/CancelJob"
AdminCommandService_PurgeHistory_FullMethodName = "/plugin.AdminCommandService/PurgeHistory"
)
// AdminCommandServiceClient is the client API for AdminCommandService service.
//
// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
//
// AdminCommandService provides administrative operations
type AdminCommandServiceClient interface {
// SaveConfig persists plugin configuration
SaveConfig(ctx context.Context, in *SaveConfigRequest, opts ...grpc.CallOption) (*SaveConfigResponse, error)
// ReloadConfig reloads configuration without restarting
ReloadConfig(ctx context.Context, in *ReloadConfigRequest, opts ...grpc.CallOption) (*ReloadConfigResponse, error)
// EnablePlugin enables a specific plugin
EnablePlugin(ctx context.Context, in *EnablePluginRequest, opts ...grpc.CallOption) (*EnablePluginResponse, error)
// DisablePlugin disables a specific plugin
DisablePlugin(ctx context.Context, in *DisablePluginRequest, opts ...grpc.CallOption) (*DisablePluginResponse, error)
// TriggerDetection manually triggers a detection for specific types
TriggerDetection(ctx context.Context, in *TriggerDetectionRequest, opts ...grpc.CallOption) (*TriggerDetectionResponse, error)
// CancelJob cancels a running job
CancelJob(ctx context.Context, in *CancelJobRequest, opts ...grpc.CallOption) (*CancelJobResponse, error)
// PurgeHistory clears job history
PurgeHistory(ctx context.Context, in *PurgeHistoryRequest, opts ...grpc.CallOption) (*PurgeHistoryResponse, error)
}
type adminCommandServiceClient struct {
cc grpc.ClientConnInterface
}
func NewAdminCommandServiceClient(cc grpc.ClientConnInterface) AdminCommandServiceClient {
return &adminCommandServiceClient{cc}
}
func (c *adminCommandServiceClient) SaveConfig(ctx context.Context, in *SaveConfigRequest, opts ...grpc.CallOption) (*SaveConfigResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(SaveConfigResponse)
err := c.cc.Invoke(ctx, AdminCommandService_SaveConfig_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *adminCommandServiceClient) ReloadConfig(ctx context.Context, in *ReloadConfigRequest, opts ...grpc.CallOption) (*ReloadConfigResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(ReloadConfigResponse)
err := c.cc.Invoke(ctx, AdminCommandService_ReloadConfig_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *adminCommandServiceClient) EnablePlugin(ctx context.Context, in *EnablePluginRequest, opts ...grpc.CallOption) (*EnablePluginResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(EnablePluginResponse)
err := c.cc.Invoke(ctx, AdminCommandService_EnablePlugin_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *adminCommandServiceClient) DisablePlugin(ctx context.Context, in *DisablePluginRequest, opts ...grpc.CallOption) (*DisablePluginResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(DisablePluginResponse)
err := c.cc.Invoke(ctx, AdminCommandService_DisablePlugin_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *adminCommandServiceClient) TriggerDetection(ctx context.Context, in *TriggerDetectionRequest, opts ...grpc.CallOption) (*TriggerDetectionResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(TriggerDetectionResponse)
err := c.cc.Invoke(ctx, AdminCommandService_TriggerDetection_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *adminCommandServiceClient) CancelJob(ctx context.Context, in *CancelJobRequest, opts ...grpc.CallOption) (*CancelJobResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(CancelJobResponse)
err := c.cc.Invoke(ctx, AdminCommandService_CancelJob_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *adminCommandServiceClient) PurgeHistory(ctx context.Context, in *PurgeHistoryRequest, opts ...grpc.CallOption) (*PurgeHistoryResponse, error) {
cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...)
out := new(PurgeHistoryResponse)
err := c.cc.Invoke(ctx, AdminCommandService_PurgeHistory_FullMethodName, in, out, cOpts...)
if err != nil {
return nil, err
}
return out, nil
}
// AdminCommandServiceServer is the server API for AdminCommandService service.
// All implementations must embed UnimplementedAdminCommandServiceServer
// for forward compatibility.
//
// AdminCommandService provides administrative operations
type AdminCommandServiceServer interface {
// SaveConfig persists plugin configuration
SaveConfig(context.Context, *SaveConfigRequest) (*SaveConfigResponse, error)
// ReloadConfig reloads configuration without restarting
ReloadConfig(context.Context, *ReloadConfigRequest) (*ReloadConfigResponse, error)
// EnablePlugin enables a specific plugin
EnablePlugin(context.Context, *EnablePluginRequest) (*EnablePluginResponse, error)
// DisablePlugin disables a specific plugin
DisablePlugin(context.Context, *DisablePluginRequest) (*DisablePluginResponse, error)
// TriggerDetection manually triggers a detection for specific types
TriggerDetection(context.Context, *TriggerDetectionRequest) (*TriggerDetectionResponse, error)
// CancelJob cancels a running job
CancelJob(context.Context, *CancelJobRequest) (*CancelJobResponse, error)
// PurgeHistory clears job history
PurgeHistory(context.Context, *PurgeHistoryRequest) (*PurgeHistoryResponse, error)
mustEmbedUnimplementedAdminCommandServiceServer()
}
// UnimplementedAdminCommandServiceServer must be embedded to have
// forward compatible implementations.
//
// NOTE: this should be embedded by value instead of pointer to avoid a nil
// pointer dereference when methods are called.
type UnimplementedAdminCommandServiceServer struct{}
func (UnimplementedAdminCommandServiceServer) SaveConfig(context.Context, *SaveConfigRequest) (*SaveConfigResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method SaveConfig not implemented")
}
func (UnimplementedAdminCommandServiceServer) ReloadConfig(context.Context, *ReloadConfigRequest) (*ReloadConfigResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method ReloadConfig not implemented")
}
func (UnimplementedAdminCommandServiceServer) EnablePlugin(context.Context, *EnablePluginRequest) (*EnablePluginResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method EnablePlugin not implemented")
}
func (UnimplementedAdminCommandServiceServer) DisablePlugin(context.Context, *DisablePluginRequest) (*DisablePluginResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method DisablePlugin not implemented")
}
func (UnimplementedAdminCommandServiceServer) TriggerDetection(context.Context, *TriggerDetectionRequest) (*TriggerDetectionResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method TriggerDetection not implemented")
}
func (UnimplementedAdminCommandServiceServer) CancelJob(context.Context, *CancelJobRequest) (*CancelJobResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method CancelJob not implemented")
}
func (UnimplementedAdminCommandServiceServer) PurgeHistory(context.Context, *PurgeHistoryRequest) (*PurgeHistoryResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method PurgeHistory not implemented")
}
func (UnimplementedAdminCommandServiceServer) mustEmbedUnimplementedAdminCommandServiceServer() {}
func (UnimplementedAdminCommandServiceServer) testEmbeddedByValue() {}
// UnsafeAdminCommandServiceServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to AdminCommandServiceServer will
// result in compilation errors.
type UnsafeAdminCommandServiceServer interface {
mustEmbedUnimplementedAdminCommandServiceServer()
}
func RegisterAdminCommandServiceServer(s grpc.ServiceRegistrar, srv AdminCommandServiceServer) {
// If the following call pancis, it indicates UnimplementedAdminCommandServiceServer was
// embedded by pointer and is nil. This will cause panics if an
// unimplemented method is ever invoked, so we test this at initialization
// time to prevent it from happening at runtime later due to I/O.
if t, ok := srv.(interface{ testEmbeddedByValue() }); ok {
t.testEmbeddedByValue()
}
s.RegisterService(&AdminCommandService_ServiceDesc, srv)
}
func _AdminCommandService_SaveConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(SaveConfigRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminCommandServiceServer).SaveConfig(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminCommandService_SaveConfig_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminCommandServiceServer).SaveConfig(ctx, req.(*SaveConfigRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AdminCommandService_ReloadConfig_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ReloadConfigRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminCommandServiceServer).ReloadConfig(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminCommandService_ReloadConfig_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminCommandServiceServer).ReloadConfig(ctx, req.(*ReloadConfigRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AdminCommandService_EnablePlugin_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(EnablePluginRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminCommandServiceServer).EnablePlugin(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminCommandService_EnablePlugin_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminCommandServiceServer).EnablePlugin(ctx, req.(*EnablePluginRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AdminCommandService_DisablePlugin_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(DisablePluginRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminCommandServiceServer).DisablePlugin(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminCommandService_DisablePlugin_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminCommandServiceServer).DisablePlugin(ctx, req.(*DisablePluginRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AdminCommandService_TriggerDetection_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(TriggerDetectionRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminCommandServiceServer).TriggerDetection(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminCommandService_TriggerDetection_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminCommandServiceServer).TriggerDetection(ctx, req.(*TriggerDetectionRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AdminCommandService_CancelJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(CancelJobRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminCommandServiceServer).CancelJob(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminCommandService_CancelJob_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminCommandServiceServer).CancelJob(ctx, req.(*CancelJobRequest))
}
return interceptor(ctx, in, info, handler)
}
func _AdminCommandService_PurgeHistory_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(PurgeHistoryRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(AdminCommandServiceServer).PurgeHistory(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: AdminCommandService_PurgeHistory_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(AdminCommandServiceServer).PurgeHistory(ctx, req.(*PurgeHistoryRequest))
}
return interceptor(ctx, in, info, handler)
}
// AdminCommandService_ServiceDesc is the grpc.ServiceDesc for AdminCommandService service.
// It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy)
var AdminCommandService_ServiceDesc = grpc.ServiceDesc{
ServiceName: "plugin.AdminCommandService",
HandlerType: (*AdminCommandServiceServer)(nil),
Methods: []grpc.MethodDesc{
{
MethodName: "SaveConfig",
Handler: _AdminCommandService_SaveConfig_Handler,
},
{
MethodName: "ReloadConfig",
Handler: _AdminCommandService_ReloadConfig_Handler,
},
{
MethodName: "EnablePlugin",
Handler: _AdminCommandService_EnablePlugin_Handler,
},
{
MethodName: "DisablePlugin",
Handler: _AdminCommandService_DisablePlugin_Handler,
},
{
MethodName: "TriggerDetection",
Handler: _AdminCommandService_TriggerDetection_Handler,
},
{
MethodName: "CancelJob",
Handler: _AdminCommandService_CancelJob_Handler,
},
{
MethodName: "PurgeHistory",
Handler: _AdminCommandService_PurgeHistory_Handler,
},
},
Streams: []grpc.StreamDesc{},
Metadata: "plugin.proto",
}
+10 -69
View File
@@ -22,7 +22,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
// Import KMS providers to register them
_ "github.com/seaweedfs/seaweedfs/weed/kms/aws"
@@ -55,7 +54,7 @@ type IdentityAccessManagement struct {
domain string
isAuthEnabled bool
credentialManager *credential.CredentialManager
filerClient *wdclient.FilerClient
filerClient filer_pb.SeaweedFilerClient
grpcDialOption grpc.DialOption
// IAM Integration for advanced features
@@ -133,37 +132,15 @@ func (c *Credential) isCredentialExpired() bool {
return c.Expiration > 0 && c.Expiration < time.Now().Unix()
}
// NewIdentityAccessManagement creates a new IAM manager
// SetFilerClient updates the filer client and its associated credential store
func (iam *IdentityAccessManagement) SetFilerClient(filerClient *wdclient.FilerClient) {
iam.m.Lock()
iam.filerClient = filerClient
iam.m.Unlock()
if iam.credentialManager == nil || filerClient == nil {
return
}
// Update credential store to use FilerClient's current filer for HA
if store := iam.credentialManager.GetStore(); store != nil {
if filerFuncSetter, ok := store.(interface {
SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption)
}); ok {
filerFuncSetter.SetFilerAddressFunc(filerClient.GetCurrentFiler, iam.grpcDialOption)
}
}
func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement {
return NewIdentityAccessManagementWithStore(option, "")
}
func NewIdentityAccessManagement(option *S3ApiServerOption, filerClient *wdclient.FilerClient) *IdentityAccessManagement {
return NewIdentityAccessManagementWithStore(option, filerClient, "")
}
func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient *wdclient.FilerClient, explicitStore string) *IdentityAccessManagement {
func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitStore string) *IdentityAccessManagement {
iam := &IdentityAccessManagement{
domain: option.DomainName,
hashes: make(map[string]*sync.Pool),
hashCounters: make(map[string]*int32),
filerClient: filerClient,
}
// Always initialize credential manager with fallback to defaults
@@ -195,25 +172,6 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient
iam.credentialManager = credentialManager
iam.stopChan = make(chan struct{})
iam.grpcDialOption = option.GrpcDialOption
// Initialize default anonymous identity
// This ensures consistent behavior for anonymous access:
// 1. In simple auth mode (no IAM integration):
// - lookupAnonymous returns this identity
// - VerifyActionPermission checks actions (which are empty) -> Denies access
// - This preserves the secure-by-default behavior for simple auth
// 2. In advanced IAM mode (with Policy Engine):
// - lookupAnonymous returns this identity
// - VerifyActionPermission proceeds to Policy Engine
// - Policy Engine evaluates against policies (DefaultEffect=Allow if no config)
// - This enables the flexible "Open by Default" for zero-config startup
iam.identityAnonymous = &Identity{
Name: "anonymous",
Account: &AccountAnonymous,
Actions: []Action{},
IsStatic: true,
}
// First, try to load configurations from file or filer
startConfigFile := option.Config
@@ -594,16 +552,6 @@ func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3
}
}
// Ensure anonymous identity exists
if identityAnonymous == nil {
identityAnonymous = &Identity{
Name: "anonymous",
Account: accounts[AccountAnonymous.Id],
Actions: []Action{},
IsStatic: true,
}
}
// atomically switch
iam.identities = identities
iam.identityAnonymous = identityAnonymous
@@ -624,9 +572,6 @@ func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3
}
}
if !exists {
if len(envIdent.Credentials) == 0 {
continue
}
iam.identities = append(iam.identities, envIdent)
iam.accessKeyIdent[envIdent.Credentials[0].AccessKey] = envIdent
iam.nameToIdentity[envIdent.Name] = envIdent
@@ -1047,8 +992,7 @@ func (iam *IdentityAccessManagement) LookupByAccessKey(accessKey string) (identi
return iam.lookupByAccessKey(accessKey)
}
// LookupAnonymous returns the anonymous identity if it exists
func (iam *IdentityAccessManagement) LookupAnonymous() (identity *Identity, found bool) {
func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) {
iam.m.RLock()
defer iam.m.RUnlock()
if iam.identityAnonymous != nil {
@@ -1168,9 +1112,6 @@ func (iam *IdentityAccessManagement) handleAuthResult(w http.ResponseWriter, r *
// Wrapper to maintain backward compatibility
func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) {
identity, err, _ := iam.authRequestWithAuthType(r, action)
if err != s3err.ErrNone {
return nil, err
}
return identity, err
}
@@ -1232,7 +1173,7 @@ func (iam *IdentityAccessManagement) authenticateRequestInternal(r *http.Request
}
case authTypeAnonymous:
amzAuthType = "Anonymous"
if identity, found = iam.LookupAnonymous(); !found {
if identity, found = iam.lookupAnonymous(); !found {
r.Header.Set(s3_constants.AmzAuthType, amzAuthType)
return identity, s3err.ErrAccessDenied, reqAuthType
}
@@ -1271,8 +1212,8 @@ func (iam *IdentityAccessManagement) authRequestWithAuthType(r *http.Request, ac
// through buckets and checking permissions for each. Skip the global check here.
policyAllows := false
if action == s3_constants.ACTION_LIST && bucket == "" && identity.Name != s3_constants.AccountAnonymousId {
// ListBuckets operation for authenticated users - authorization handled per-bucket in the handler
if action == s3_constants.ACTION_LIST && bucket == "" {
// ListBuckets operation - authorization handled per-bucket in the handler
} else {
// First check bucket policy if one exists
// Bucket policies can grant or deny access to specific users/principals
@@ -1366,8 +1307,8 @@ func (iam *IdentityAccessManagement) AuthSignatureOnly(r *http.Request) (*Identi
return identity, s3err.ErrNotImplemented
}
case authTypeAnonymous:
// Anonymous users can be authenticated, but authorization is handled separately
return iam.identityAnonymous, s3err.ErrNone
// Anonymous users cannot use IAM API
return identity, s3err.ErrAccessDenied
default:
return identity, s3err.ErrNotImplemented
}
+3 -3
View File
@@ -450,7 +450,7 @@ func TestNewIdentityAccessManagementWithStoreEnvVars(t *testing.T) {
option := &S3ApiServerOption{
Config: "", // No config file - this should trigger environment variable fallback
}
iam := NewIdentityAccessManagementWithStore(option, nil, string(credential.StoreTypeMemory))
iam := NewIdentityAccessManagementWithStore(option, string(credential.StoreTypeMemory))
if tt.expectEnvIdentity {
// Should have exactly one identity from environment variables
@@ -510,7 +510,7 @@ func TestConfigFileWithNoIdentitiesAllowsEnvVars(t *testing.T) {
option := &S3ApiServerOption{
Config: tmpFile.Name(),
}
iam := NewIdentityAccessManagementWithStore(option, nil, string(credential.StoreTypeMemory))
iam := NewIdentityAccessManagementWithStore(option, string(credential.StoreTypeMemory))
// Should have exactly one identity from environment variables
assert.Len(t, iam.identities, 1, "Should have exactly one identity from environment variables even when config file exists with no identities")
@@ -762,7 +762,7 @@ func TestSignatureVerificationDoesNotCheckPermissions(t *testing.T) {
}
func TestStaticIdentityProtection(t *testing.T) {
iam := NewIdentityAccessManagement(&S3ApiServerOption{}, nil)
iam := NewIdentityAccessManagement(&S3ApiServerOption{})
// Add a static identity
staticIdent := &Identity{
+1 -1
View File
@@ -66,7 +66,7 @@ func TestReproIssue7912(t *testing.T) {
option := &S3ApiServerOption{
Config: tmpFile.Name(),
}
iam := NewIdentityAccessManagementWithStore(option, nil, "memory")
iam := NewIdentityAccessManagementWithStore(option, "memory")
assert.True(t, iam.isEnabled(), "Auth should be enabled")
-4
View File
@@ -44,10 +44,6 @@ func (m *MockIAMIntegration) ValidateTrustPolicyForPrincipal(ctx context.Context
return nil
}
func (m *MockIAMIntegration) DefaultAllow() bool {
return true
}
// TestVerifyV4SignatureWithSTSIdentity tests that verifyV4Signature properly handles STS identities
// by falling back to IAM authorization when shouldCheckPermissions is true
func TestVerifyV4SignatureWithSTSIdentity(t *testing.T) {
+3 -3
View File
@@ -22,7 +22,7 @@ func TestSTSIdentityPolicyNamesPopulation(t *testing.T) {
stsService, config := setupTestSTSService(t)
// Create IAM with STS integration
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, "memory")
s3iam := &S3IAMIntegration{
stsService: stsService,
}
@@ -264,7 +264,7 @@ func TestValidateSTSSessionTokenIntegration(t *testing.T) {
stsService, config := setupTestSTSService(t)
// Create IAM with STS integration
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, "memory")
s3iam := &S3IAMIntegration{
stsService: stsService,
}
@@ -311,7 +311,7 @@ func TestSTSIdentityClaimsPopulation(t *testing.T) {
stsService, config := setupTestSTSService(t)
// Create IAM with STS integration
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, "memory")
s3iam := &S3IAMIntegration{
stsService: stsService,
}
-156
View File
@@ -1,156 +0,0 @@
package s3api
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadIAMManagerFromConfig_Defaults(t *testing.T) {
// Create a temporary config file with minimal content (just policy)
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "iam_config.json")
configContent := `{
"sts": {
"providers": []
},
"policy": {
"storeType": "memory",
"defaultEffect": "Allow"
}
}`
err := os.WriteFile(configPath, []byte(configContent), 0644)
assert.NoError(t, err)
// dummy filer address provider
filerProvider := func() string { return "localhost:8888" }
defaultSigningKeyProvider := func() string { return "default-secure-signing-key" }
// Load the manager
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
assert.NoError(t, err)
assert.NotNil(t, manager)
}
func TestLoadIAMManagerFromConfig_Overrides(t *testing.T) {
// Create a temporary config file with EXPLICIT values
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "iam_config_explicit.json")
configContent := `{
"sts": {
"tokenDuration": "2h",
"maxSessionLength": "24h",
"issuer": "custom-issuer",
"signingKey": "ZXhwbGljaXQtc2lnbmluZy1rZXktMTIzNDU="
},
"policy": {
"storeType": "memory",
"defaultEffect": "Allow"
}
}`
// Base64 encoded "explicit-signing-key-12345" is "ZXhwbGljaXQtc2lnbmluZy1rZXktMTIzNDU="
err := os.WriteFile(configPath, []byte(configContent), 0644)
assert.NoError(t, err)
filerProvider := func() string { return "localhost:8888" }
defaultSigningKeyProvider := func() string { return "default-secure-signing-key" }
// Load
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
assert.NoError(t, err)
assert.NotNil(t, manager)
}
func TestLoadIAMManagerFromConfig_PartialDefaults(t *testing.T) {
// Test that partial configs (e.g. providing SigningKey but not Duration) work
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "iam_config_partial.json")
// Signing key provided in JSON, others missing
configContent := `{
"sts": {
"signingKey": "anNvbi1wcm92aWRlZC1rZXktMTIzNDU="
},
"policy": {
"storeType": "memory",
"defaultEffect": "Allow"
}
}`
err := os.WriteFile(configPath, []byte(configContent), 0644)
assert.NoError(t, err)
filerProvider := func() string { return "localhost:8888" }
// Default signing key provided but should be IGNORED because JSON has one
defaultSigningKeyProvider := func() string { return "server-default-key-should-be-ignored" }
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
assert.NoError(t, err)
assert.NotNil(t, manager)
}
func TestLoadIAMManagerFromConfig_ExplicitEmptyKey(t *testing.T) {
// Test that if JSON has empty signing key string, it still falls back
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "iam_config_empty_key.json")
// Signing key explicitly empty
configContent := `{
"sts": {
"signingKey": ""
},
"policy": {
"storeType": "memory",
"defaultEffect": "Allow"
}
}`
err := os.WriteFile(configPath, []byte(configContent), 0644)
assert.NoError(t, err)
filerProvider := func() string { return "localhost:8888" }
defaultSigningKeyProvider := func() string { return "fallback-key-should-be-used" }
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
assert.NoError(t, err)
assert.NotNil(t, manager)
}
func TestLoadIAMManagerFromConfig_MissingKeyError(t *testing.T) {
// Test that if BOTH keys are empty, it fails with a clear error
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "iam_config_all_empty.json")
// Signing key explicitly empty in JSON
configContent := `{
"sts": {
"signingKey": ""
},
"policy": {
"storeType": "memory",
"defaultEffect": "Allow"
}
}`
err := os.WriteFile(configPath, []byte(configContent), 0644)
assert.NoError(t, err)
filerProvider := func() string { return "localhost:8888" }
defaultSigningKeyProvider := func() string { return "" } // Empty default too
// Ensure no SSE-S3 key interferes (global state in tests is tricky, but let's assume clean state or no mock)
// Ideally we would mock GetSSES3KeyManager().GetMasterKey() but it's a global singleton.
// For this unit test, if the global key manager has no key, it should fail.
_, err = loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
// Should return a clear error
assert.Error(t, err)
assert.Contains(t, err.Error(), "no signing key found for STS service")
}
-50
View File
@@ -1,50 +0,0 @@
package s3api
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLoadIAMManagerFromConfig_OptionalConfig(t *testing.T) {
// Mock dependencies
filerAddressProvider := func() string { return "localhost:8888" }
getFilerSigningKey := func() string { return "test-signing-key" }
// Test Case 1: Empty config path should load defaults
iamManager, err := loadIAMManagerFromConfig("", filerAddressProvider, getFilerSigningKey)
require.NoError(t, err)
require.NotNil(t, iamManager)
// Verify STS Service is initialized with defaults
stsService := iamManager.GetSTSService()
assert.NotNil(t, stsService)
// Verify defaults are applied
// Since we can't easily access the internal config of stsService,
// we rely on the fact that initialization succeeded without error.
// We can also verify that the policy engine uses memory store by default.
// Verify Policy Engine is initialized with defaults (Memory store, Deny effect)
// Again, internal state might be hard to access directly, but successful init implies defaults worked.
}
func TestLoadIAMManagerFromConfig_EmptyConfigWithFallbackKey(t *testing.T) {
// Mock dependencies where getFilerSigningKey returns empty, forcing fallback logic
// Initialize IAM with empty config (should trigger defaults)
// We pass empty string for config file path
option := &S3ApiServerOption{
Config: "",
IamConfig: "",
EnableIam: true,
}
iamManager := NewIdentityAccessManagementWithStore(option, nil, "memory")
// Verify identityAnonymous is initialized
// This confirms the fix for anonymous access in zero-config mode
anonIdentity, found := iamManager.LookupAnonymous()
assert.True(t, found, "Anonymous identity should be found by default")
assert.NotNil(t, anonIdentity, "Anonymous identity should not be nil")
assert.Equal(t, "anonymous", anonIdentity.Name)
}
+13 -19
View File
@@ -18,7 +18,6 @@ type FilerClient interface {
type S3Authenticator interface {
AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode)
DefaultAllow() bool
}
// Server implements the Iceberg REST Catalog API.
@@ -129,25 +128,20 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
identityName, identity, errCode := s.authenticator.AuthenticateRequest(r)
if errCode != s3err.ErrNone {
// If authentication failed but DefaultAllow is enabled, proceed without identity
if s.authenticator.DefaultAllow() {
glog.V(2).Infof("Iceberg: AuthenticateRequest failed (%v), but DefaultAllow is true, proceeding", errCode)
} else {
apiErr := s3err.GetAPIError(errCode)
errorType := "RESTException"
switch apiErr.HTTPStatusCode {
case http.StatusForbidden:
errorType = "ForbiddenException"
case http.StatusUnauthorized:
errorType = "NotAuthorizedException"
case http.StatusBadRequest:
errorType = "BadRequestException"
case http.StatusInternalServerError:
errorType = "InternalServerError"
}
writeError(w, apiErr.HTTPStatusCode, errorType, apiErr.Description)
return
apiErr := s3err.GetAPIError(errCode)
errorType := "RESTException"
switch apiErr.HTTPStatusCode {
case http.StatusForbidden:
errorType = "ForbiddenException"
case http.StatusUnauthorized:
errorType = "NotAuthorizedException"
case http.StatusBadRequest:
errorType = "BadRequestException"
case http.StatusInternalServerError:
errorType = "InternalServerError"
}
writeError(w, apiErr.HTTPStatusCode, errorType, apiErr.Description)
return
}
if identityName != "" || identity != nil {
-9
View File
@@ -44,7 +44,6 @@ type IAMIntegration interface {
AuthorizeAction(ctx context.Context, identity *IAMIdentity, action Action, bucket string, objectKey string, r *http.Request) s3err.ErrorCode
ValidateSessionToken(ctx context.Context, token string) (*sts.SessionInfo, error)
ValidateTrustPolicyForPrincipal(ctx context.Context, roleArn, principalArn string) error
DefaultAllow() bool
}
// S3IAMIntegration provides IAM integration for S3 API
@@ -311,14 +310,6 @@ func (s3iam *S3IAMIntegration) ValidateTrustPolicyForPrincipal(ctx context.Conte
return s3iam.iamManager.ValidateTrustPolicyForPrincipal(ctx, roleArn, principalArn)
}
// DefaultAllow returns whether access is allowed by default when no policy is found
func (s3iam *S3IAMIntegration) DefaultAllow() bool {
if s3iam.iamManager == nil {
return true // Default to true if IAM is not enabled
}
return s3iam.iamManager.DefaultAllow()
}
// IAMIdentity represents an authenticated identity with session information
type IAMIdentity struct {
Name string
+2 -50
View File
@@ -5,7 +5,6 @@ import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
@@ -20,13 +19,9 @@ import (
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"golang.org/x/crypto/hkdf"
"google.golang.org/grpc"
)
// SSE-S3 uses AES-256 encryption with server-managed keys
@@ -457,27 +452,6 @@ func (km *SSES3KeyManager) GetKey(keyID string) (*SSES3Key, bool) {
return nil, false
}
// GetMasterKey returns a derived key from the master KEK for STS signing
// This uses HKDF to isolate the STS security domain from the SSE-S3 domain
func (km *SSES3KeyManager) GetMasterKey() []byte {
km.mu.RLock()
defer km.mu.RUnlock()
if len(km.superKey) == 0 {
return nil
}
// Derive a separate key for STS to isolate security domains
// We use the KEK as the secret, and "seaweedfs-sts-signing-key" as the info
hkdfReader := hkdf.New(sha256.New, km.superKey, nil, []byte("seaweedfs-sts-signing-key"))
derived := make([]byte, 32) // 256-bit derived key
if _, err := io.ReadFull(hkdfReader, derived); err != nil {
glog.Errorf("Failed to derive STS key: %v", err)
return nil
}
return derived
}
// Global SSE-S3 key manager instance
var globalSSES3KeyManager = NewSSES3KeyManager()
@@ -486,31 +460,9 @@ func GetSSES3KeyManager() *SSES3KeyManager {
return globalSSES3KeyManager
}
// KeyManagerFilerClient wraps wdclient.FilerClient to satisfy filer_pb.FilerClient interface
type KeyManagerFilerClient struct {
*wdclient.FilerClient
grpcDialOption grpc.DialOption
}
func (k *KeyManagerFilerClient) AdjustedUrl(location *filer_pb.Location) string {
return location.Url
}
func (k *KeyManagerFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
filerAddress := k.GetCurrentFiler()
if filerAddress == "" {
return fmt.Errorf("no filer available")
}
return pb.WithGrpcFilerClient(streamingMode, 0, filerAddress, k.grpcDialOption, fn)
}
// InitializeGlobalSSES3KeyManager initializes the global key manager with filer access
func InitializeGlobalSSES3KeyManager(filerClient *wdclient.FilerClient, grpcDialOption grpc.DialOption) error {
wrapper := &KeyManagerFilerClient{
FilerClient: filerClient,
grpcDialOption: grpcDialOption,
}
return globalSSES3KeyManager.InitializeWithFiler(wrapper)
func InitializeGlobalSSES3KeyManager(s3ApiServer *S3ApiServer) error {
return globalSSES3KeyManager.InitializeWithFiler(s3ApiServer)
}
// ProcessSSES3Request processes an SSE-S3 request and returns encryption metadata
+37 -41
View File
@@ -19,6 +19,21 @@ const (
deleteMultipleObjectsLimit = 1000
)
// objectLockVersionToCheckForDelete resolves which version should be validated for Object Lock protections.
// For versioned delete without explicit versionId, this targets the latest version (empty versionId for enabled,
// "null" for suspended) because DeleteObject affects that version's visibility.
// This always returns a version ID to check; object lock protections are fail-closed.
func objectLockVersionToCheckForDelete(versioningState, requestedVersionID string) string {
if requestedVersionID != "" {
return requestedVersionID
}
if versioningState == s3_constants.VersioningSuspended {
return "null"
}
return ""
}
func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
bucket, object := s3_constants.GetBucketAndObject(r)
@@ -52,18 +67,18 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
}
lockCheckVersionID := objectLockVersionToCheckForDelete(versioningState, versionId)
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, lockCheckVersionID, governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object, lockCheckVersionID, err)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
if versioningConfigured {
// Handle versioned delete based on specific versioning state
if versionId != "" {
// Delete specific version (same for both enabled and suspended)
// Check object lock permissions before deleting specific version
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
// Delete specific version
err := s3a.deleteSpecificObjectVersion(bucket, object, versionId)
if err != nil {
@@ -77,9 +92,7 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
} else {
// Delete without version ID - behavior depends on versioning state
if versioningEnabled {
// Enabled versioning: Create delete marker (logical delete)
// AWS S3 behavior: Delete marker creation is NOT blocked by object retention
// because it's a logical delete that doesn't actually remove the retained version
// Enabled versioning: create delete marker (logical delete)
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
if err != nil {
glog.Errorf("Failed to create delete marker: %v", err)
@@ -94,14 +107,6 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
// Suspended versioning: Actually delete the "null" version object
glog.V(2).Infof("DeleteObjectHandler: deleting null version for suspended versioning %s/%s", bucket, object)
// Check object lock permissions before deleting "null" version
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, "null", governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
// Delete the "null" version (the regular file)
err := s3a.deleteSpecificObjectVersion(bucket, object, "null")
if err != nil {
@@ -116,14 +121,6 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
}
} else {
// Handle regular delete (non-versioned)
// Check object lock permissions before deleting object
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
// Normalize trailing-slash object keys (e.g. "path/") to the
// underlying directory entry path so DeleteEntry gets a valid name.
target := util.NewFullPath(s3a.bucketDir(bucket), object)
@@ -249,20 +246,19 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
continue
}
// Check object lock permissions before deletion (only for versioned buckets)
if versioningConfigured {
// Validate governance bypass for this specific object
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key)
if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, object.VersionId, governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object.Key, object.VersionId, err)
deleteErrors = append(deleteErrors, DeleteError{
Code: s3err.GetAPIError(s3err.ErrAccessDenied).Code,
Message: s3err.GetAPIError(s3err.ErrAccessDenied).Description,
Key: object.Key,
VersionId: object.VersionId,
})
continue
}
// Check object lock permissions before deletion (applies to all buckets: versioned or non-versioned)
lockCheckVersionID := objectLockVersionToCheckForDelete(versioningState, object.VersionId)
// Validate governance bypass for this specific object
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key)
if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, lockCheckVersionID, governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object.Key, lockCheckVersionID, err)
deleteErrors = append(deleteErrors, DeleteError{
Code: s3err.GetAPIError(s3err.ErrAccessDenied).Code,
Message: s3err.GetAPIError(s3err.ErrAccessDenied).Description,
Key: object.Key,
VersionId: object.VersionId,
})
continue
}
var deleteVersionId string
@@ -0,0 +1,61 @@
package s3api
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/stretchr/testify/assert"
)
func TestObjectLockVersionToCheckForDelete(t *testing.T) {
tests := []struct {
name string
versioningState string
requestedVersionID string
expectedVersionID string
}{
{
name: "enabled versioning without version id checks latest version",
versioningState: s3_constants.VersioningEnabled,
requestedVersionID: "",
expectedVersionID: "",
},
{
name: "suspended versioning without version id checks null version",
versioningState: s3_constants.VersioningSuspended,
requestedVersionID: "",
expectedVersionID: "null",
},
{
name: "specific version id is always checked",
versioningState: s3_constants.VersioningEnabled,
requestedVersionID: "3LgYQ7f7VxQ3",
expectedVersionID: "3LgYQ7f7VxQ3",
},
{
name: "non-versioned buckets still check current object",
versioningState: "",
requestedVersionID: "",
expectedVersionID: "",
},
{
name: "unknown versioning state defaults to empty version",
versioningState: "UnexpectedState",
requestedVersionID: "",
expectedVersionID: "",
},
{
name: "suspended versioning with specific version id checks that version",
versioningState: s3_constants.VersioningSuspended,
requestedVersionID: "abc123",
expectedVersionID: "abc123",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
versionID := objectLockVersionToCheckForDelete(tc.versioningState, tc.requestedVersionID)
assert.Equal(t, tc.expectedVersionID, versionID)
})
}
}
+3 -3
View File
@@ -13,7 +13,7 @@ import (
func TestGetRequestDataReader_ChunkedEncodingWithoutIAM(t *testing.T) {
// Create an S3ApiServer with IAM disabled
s3a := &S3ApiServer{
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, string(credential.StoreTypeMemory)),
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
}
// Ensure IAM is disabled for this test
s3a.iam.isAuthEnabled = false
@@ -87,7 +87,7 @@ func TestGetRequestDataReader_ChunkedEncodingWithoutIAM(t *testing.T) {
func TestGetRequestDataReader_AuthTypeDetection(t *testing.T) {
// Create an S3ApiServer with IAM disabled
s3a := &S3ApiServer{
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, string(credential.StoreTypeMemory)),
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
}
s3a.iam.isAuthEnabled = false
@@ -122,7 +122,7 @@ func TestGetRequestDataReader_AuthTypeDetection(t *testing.T) {
func TestGetRequestDataReader_IAMEnabled(t *testing.T) {
// Create an S3ApiServer with IAM enabled
s3a := &S3ApiServer{
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, string(credential.StoreTypeMemory)),
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
}
s3a.iam.isAuthEnabled = true
+40 -94
View File
@@ -110,8 +110,7 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
option.AllowedOrigins = domains
}
// Initialize basic/legacy IAM - filerClient not available yet, passed as nil
iam := NewIdentityAccessManagementWithStore(option, nil, explicitStore)
iam := NewIdentityAccessManagementWithStore(option, explicitStore)
// Initialize bucket policy engine first
policyEngine := NewBucketPolicyEngine()
@@ -147,24 +146,16 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
glog.V(1).Infof("S3 API initialized FilerClient with %d filer(s) (no discovery)", len(option.Filers))
}
// Initialize Global SSE-S3 Key Manager early so it's available for IAM fallback
// This ensures we can access the KEK for STS signing key if needed
if err := InitializeGlobalSSES3KeyManager(filerClient, option.GrpcDialOption); err != nil {
glog.Errorf("Failed to initialize SSE-S3 Key Manager: %v", err)
// We continue, as this might be a transient failure or non-critical for some setups,
// but IAM fallback to KEK will fail if this didn't succeed.
}
// Update credential store to use FilerClient's current filer for HA
iam.SetFilerClient(filerClient)
// Keep attempting to load configuration from filer now that we have a client
// The initial load in NewIdentityAccessManagementWithStore might have failed if client was nil
go func() {
if err := iam.loadS3ApiConfigurationFromFiler(option); err != nil {
glog.Warningf("Failed to load IAM config from filer after client update: %v", err)
if store := iam.credentialManager.GetStore(); store != nil {
if filerFuncSetter, ok := store.(interface {
SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption)
}); ok {
// Use FilerClient's GetCurrentFiler for true HA
filerFuncSetter.SetFilerAddressFunc(filerClient.GetCurrentFiler, option.GrpcDialOption)
glog.V(1).Infof("Updated credential store to use FilerClient's current active filer (HA-aware)")
}
}()
}
s3ApiServer = &S3ApiServer{
option: option,
@@ -187,25 +178,19 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
// This avoids circular dependency by not passing the entire S3ApiServer
iam.policyEngine = policyEngine
// Initialize advanced IAM system if config is provided or explicitly enabled
if option.IamConfig != "" || option.EnableIam {
configSource := "defaults"
if option.IamConfig != "" {
configSource = option.IamConfig
}
glog.V(1).Infof("Loading advanced IAM configuration from: %s", configSource)
// Initialize advanced IAM system if config is provided
if option.IamConfig != "" {
glog.V(1).Infof("Loading advanced IAM configuration from: %s", option.IamConfig)
// Use FilerClient's GetCurrentFiler for HA-aware filer selection
iamManager, err := loadIAMManagerFromConfig(option.IamConfig, func() string {
return string(filerClient.GetCurrentFiler())
}, func() string {
return signingKey
})
if err != nil {
glog.Errorf("Failed to load IAM configuration: %v", err)
} else {
if s3ApiServer.iam.credentialManager != nil {
iamManager.SetUserStore(s3ApiServer.iam.credentialManager)
if iam.credentialManager != nil {
iamManager.SetUserStore(iam.credentialManager)
}
glog.V(1).Infof("IAM Manager loaded, creating integration")
// Create S3 IAM integration with the loaded IAM manager
@@ -248,10 +233,6 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
})
}
s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer)
// Update IAM with the final filer client (already handled by SetFilerClient above,
// but this reinforces it if we ever change the flow)
s3ApiServer.iam.SetFilerClient(s3ApiServer.filerClient)
if option.LocalFilerSocket == "" {
if s3ApiServer.client, err = util_http.NewGlobalHttpClient(); err != nil {
return nil, err
@@ -268,6 +249,11 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
s3ApiServer.registerRouter(router)
// Initialize the global SSE-S3 key manager with filer access
if err := InitializeGlobalSSES3KeyManager(s3ApiServer); err != nil {
return nil, fmt.Errorf("failed to initialize SSE-S3 key manager: %w", err)
}
go s3ApiServer.subscribeMetaEvents("s3", startTsNs, filer.DirectoryEtcRoot, []string{
option.BucketsPath,
filer.IamConfigDirectory,
@@ -844,7 +830,14 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
}
// loadIAMManagerFromConfig loads the advanced IAM manager from configuration file
func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() string, getFilerSigningKey func() string) (*integration.IAMManager, error) {
func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() string) (*integration.IAMManager, error) {
// Read configuration file
configData, err := os.ReadFile(configPath)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
// Parse configuration structure
var configRoot struct {
STS *sts.STSConfig `json:"sts"`
Policy *policy.PolicyEngineConfig `json:"policy"`
@@ -856,43 +849,24 @@ func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() str
} `json:"policies"`
}
if configPath != "" {
// Read configuration file
configData, err := os.ReadFile(configPath)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
if err := json.Unmarshal(configData, &configRoot); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
} else {
glog.V(1).Infof("No IAM config file provided; using defaults")
// Initialize with empty config which will trigger defaults below
}
// Ensure STS config exists so we can apply defaults later
if configRoot.STS == nil {
configRoot.STS = &sts.STSConfig{}
if err := json.Unmarshal(configData, &configRoot); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
// Ensure a valid policy engine config exists
if configRoot.Policy == nil {
configRoot.Policy = &policy.PolicyEngineConfig{}
}
if configRoot.Policy.StoreType == "" {
configRoot.Policy.StoreType = sts.StoreTypeMemory
}
if configRoot.Policy.DefaultEffect == "" {
// Default to Allow (open) with in-memory store so that
// users can start using STS without locking themselves out immediately.
// For other stores (e.g. filer), default to Deny (closed) for security.
if configRoot.Policy.StoreType == sts.StoreTypeMemory {
configRoot.Policy.DefaultEffect = sts.EffectAllow
} else {
configRoot.Policy.DefaultEffect = sts.EffectDeny
// Provide a secure default if not specified in the config file
// Default to Deny with in-memory store so that JSON-defined policies work without filer
glog.V(1).Infof("No policy engine config provided; using defaults (DefaultEffect=%s, StoreType=%s)", sts.EffectDeny, sts.StoreTypeMemory)
configRoot.Policy = &policy.PolicyEngineConfig{
DefaultEffect: sts.EffectDeny,
StoreType: sts.StoreTypeMemory,
}
glog.V(1).Infof("Using policy defaults: DefaultEffect=%s, StoreType=%s", configRoot.Policy.DefaultEffect, configRoot.Policy.StoreType)
} else if configRoot.Policy.StoreType == "" {
// If policy config exists but storeType is not specified, use memory store
// This ensures JSON-defined policies are stored in memory and work correctly
configRoot.Policy.StoreType = sts.StoreTypeMemory
glog.V(1).Infof("Policy storeType not specified; using memory store for JSON config-based setup")
}
// Create IAM configuration
@@ -904,26 +878,6 @@ func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() str
},
}
// Apply default signing key if not present in config
if iamConfig.STS != nil && len(iamConfig.STS.SigningKey) == 0 {
// 1. Try server-configured signing key (security.toml / CLI)
if key := getFilerSigningKey(); key != "" {
iamConfig.STS.SigningKey = []byte(key)
glog.V(1).Infof("Using default filer signing key for STS service")
} else {
// 2. Try cluster-wide SSE-S3 Master Key (KEK) from Filer
// This ensures zero-config consistency across the cluster
if kek := GetSSES3KeyManager().GetMasterKey(); len(kek) > 0 {
iamConfig.STS.SigningKey = kek
glog.V(1).Infof("Using SSE-S3 Master Key (KEK) for STS service")
} else {
// 3. Fail if no signing key is available
// This ensures consistency across multiple S3 servers and secure operation
return nil, fmt.Errorf("no signing key found for STS service; please provide 'signingKey' in IAM config, configure 'jwt.filer_signing.key' in security.toml, or ensure SSE-S3 is initialized")
}
}
}
// Initialize IAM manager
iamManager := integration.NewIAMManager()
if err := iamManager.Initialize(iamConfig, filerAddressProvider); err != nil {
@@ -1006,11 +960,3 @@ func (s3a *S3ApiServer) AuthenticateRequest(r *http.Request) (string, interface{
}
return "", nil, err
}
// DefaultAllow returns whether access is allowed by default when no policy is found
func (s3a *S3ApiServer) DefaultAllow() bool {
if s3a.iam == nil || s3a.iam.iamIntegration == nil {
return false
}
return s3a.iam.iamIntegration.DefaultAllow()
}
+1 -1
View File
@@ -16,7 +16,7 @@ import (
// setupRoutingTestServer creates a minimal S3ApiServer for routing tests
func setupRoutingTestServer(t *testing.T) *S3ApiServer {
opt := &S3ApiServerOption{EnableIam: true}
iam := NewIdentityAccessManagementWithStore(opt, nil, "memory")
iam := NewIdentityAccessManagementWithStore(opt, "memory")
iam.isAuthEnabled = true
if iam.credentialManager == nil {
+3 -20
View File
@@ -43,11 +43,6 @@ func (st *S3TablesApiServer) SetAccountID(accountID string) {
st.handler.SetAccountID(accountID)
}
// SetDefaultAllow sets whether to allow access by default
func (st *S3TablesApiServer) SetDefaultAllow(allow bool) {
st.handler.SetDefaultAllow(allow)
}
// S3TablesHandler handles S3 Tables API requests
func (st *S3TablesApiServer) S3TablesHandler(w http.ResponseWriter, r *http.Request) {
st.handler.HandleRequest(w, r, st)
@@ -62,12 +57,6 @@ func (st *S3TablesApiServer) WithFilerClient(streamingMode bool, fn func(filer_p
func (s3a *S3ApiServer) registerS3TablesRoutes(router *mux.Router) {
// Create S3 Tables handler
s3TablesApi := NewS3TablesApiServer(s3a)
if s3a.iam != nil && s3a.iam.iamIntegration != nil {
s3TablesApi.SetDefaultAllow(s3a.iam.iamIntegration.DefaultAllow())
} else {
// If IAM is not configured, allow all access by default
s3TablesApi.SetDefaultAllow(true)
}
// Regex for S3 Tables Bucket ARN
const tableBucketARNRegex = "arn:aws:s3tables:[^/:]*:[^/:]*:bucket/[^/]+"
@@ -629,15 +618,9 @@ func (s3a *S3ApiServer) authenticateS3Tables(f http.HandlerFunc) http.HandlerFun
// Use AuthSignatureOnly to authenticate the request without authorizing specific actions
identity, errCode := s3a.iam.AuthSignatureOnly(r)
if errCode != s3err.ErrNone {
// If IAM is enabled but DefaultAllow is true, we can proceed even if unauthenticated
// authorization checks in handlers will then use DefaultAllow logic.
if s3a.iam.iamIntegration != nil && s3a.iam.iamIntegration.DefaultAllow() {
glog.V(2).Infof("S3Tables: AuthSignatureOnly failed (%v), but DefaultAllow is true, proceeding", errCode)
} else {
glog.Errorf("S3Tables: AuthSignatureOnly failed: %v", errCode)
s3err.WriteErrorResponse(w, r, errCode)
return
}
glog.Errorf("S3Tables: AuthSignatureOnly failed: %v", errCode)
s3err.WriteErrorResponse(w, r, errCode)
return
}
// Store the authenticated identity in request context
+4 -11
View File
@@ -44,17 +44,15 @@ const (
// S3TablesHandler handles S3 Tables API requests
type S3TablesHandler struct {
region string
accountID string
defaultAllow bool // Whether to allow access by default (for zero-config IAM)
region string
accountID string
}
// NewS3TablesHandler creates a new S3 Tables handler
func NewS3TablesHandler() *S3TablesHandler {
return &S3TablesHandler{
region: DefaultRegion,
accountID: DefaultAccountID,
defaultAllow: false,
region: DefaultRegion,
accountID: DefaultAccountID,
}
}
@@ -72,11 +70,6 @@ func (h *S3TablesHandler) SetAccountID(accountID string) {
}
}
// SetDefaultAllow sets whether to allow access by default
func (h *S3TablesHandler) SetDefaultAllow(allow bool) {
h.defaultAllow = allow
}
// FilerClient interface for filer operations
type FilerClient interface {
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
+1 -3
View File
@@ -16,9 +16,7 @@ import (
func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
// Check permission
principal := h.getAccountID(r)
if !CheckPermissionWithContext("CreateTableBucket", principal, principal, "", "", &PolicyContext{
DefaultAllow: h.defaultAllow,
}) {
if !CanCreateTableBucket(principal, principal, "") {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
}
@@ -72,7 +72,6 @@ func (h *S3TablesHandler) handleGetTableBucket(w http.ResponseWriter, r *http.Re
if !CheckPermissionWithContext("GetTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket details")
return ErrAccessDenied
@@ -102,7 +101,6 @@ func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("ListTableBuckets", principal, accountID, "", "", &PolicyContext{
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to list table buckets")
return NewAuthError("ListTableBuckets", principal, "not authorized to list table buckets")
@@ -200,7 +198,6 @@ func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.
if !CheckPermissionWithContext("GetTableBucket", accountID, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: entry.Entry.Name,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
continue
}
@@ -303,7 +300,6 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http
if !CheckPermissionWithContext("DeleteTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
return NewAuthError("DeleteTableBucket", principal, fmt.Sprintf("not authorized to delete bucket %s", bucketName))
}
-4
View File
@@ -118,7 +118,6 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
glog.Infof("S3Tables: Permission denied for CreateNamespace - principal=%s, owner=%s", principal, bucketMetadata.OwnerAccountID)
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create namespace in this bucket")
@@ -259,7 +258,6 @@ func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Requ
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
return ErrAccessDenied
@@ -346,7 +344,6 @@ func (h *S3TablesHandler) handleListNamespaces(w http.ResponseWriter, r *http.Re
TableBucketName: bucketName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
return ErrAccessDenied
@@ -531,7 +528,6 @@ func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.R
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
return ErrAccessDenied
-9
View File
@@ -94,7 +94,6 @@ func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *h
if !CheckPermissionWithContext("PutTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, "", bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table bucket policy")
return NewAuthError("PutTableBucketPolicy", principal, "not authorized to put table bucket policy")
@@ -172,7 +171,6 @@ func (h *S3TablesHandler) handleGetTableBucketPolicy(w http.ResponseWriter, r *h
if !CheckPermissionWithContext("GetTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, string(policy), bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket policy")
return NewAuthError("GetTableBucketPolicy", principal, "not authorized to get table bucket policy")
@@ -248,7 +246,6 @@ func (h *S3TablesHandler) handleDeleteTableBucketPolicy(w http.ResponseWriter, r
if !CheckPermissionWithContext("DeleteTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table bucket policy")
return NewAuthError("DeleteTableBucketPolicy", principal, "not authorized to delete table bucket policy")
@@ -349,7 +346,6 @@ func (h *S3TablesHandler) handlePutTablePolicy(w http.ResponseWriter, r *http.Re
Namespace: namespaceName,
TableName: tableName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table policy")
return NewAuthError("PutTablePolicy", principal, "not authorized to put table policy")
@@ -457,7 +453,6 @@ func (h *S3TablesHandler) handleGetTablePolicy(w http.ResponseWriter, r *http.Re
Namespace: namespaceName,
TableName: tableName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table policy")
return NewAuthError("GetTablePolicy", principal, "not authorized to get table policy")
@@ -547,7 +542,6 @@ func (h *S3TablesHandler) handleDeleteTablePolicy(w http.ResponseWriter, r *http
Namespace: namespaceName,
TableName: tableName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table policy")
return NewAuthError("DeleteTablePolicy", principal, "not authorized to delete table policy")
@@ -646,7 +640,6 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque
TagKeys: requestTagKeys,
ResourceTags: existingTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
return NewAuthError("TagResource", principal, "not authorized to tag resource")
}
@@ -764,7 +757,6 @@ func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *ht
TableBucketTags: bucketTags,
ResourceTags: tags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
return NewAuthError("ListTagsForResource", principal, "not authorized to list tags for resource")
}
@@ -872,7 +864,6 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req
TagKeys: req.TagKeys,
ResourceTags: tags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
return NewAuthError("UntagResource", principal, "not authorized to untag resource")
}
-11
View File
@@ -145,7 +145,6 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
TagKeys: mapKeys(req.Tags),
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
bucketAllowed := CheckPermissionWithContext("CreateTable", accountID, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
@@ -155,7 +154,6 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
TagKeys: mapKeys(req.Tags),
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
if !nsAllowed && !bucketAllowed {
@@ -392,7 +390,6 @@ func (h *S3TablesHandler) handleGetTable(w http.ResponseWriter, r *http.Request,
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
bucketAllowed := CheckPermissionWithContext("GetTable", accountID, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
@@ -401,7 +398,6 @@ func (h *S3TablesHandler) handleGetTable(w http.ResponseWriter, r *http.Request,
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
if !tableAllowed && !bucketAllowed {
@@ -531,14 +527,12 @@ func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Reques
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
bucketAllowed := CheckPermissionWithContext("ListTables", accountID, bucketMeta.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
if !nsAllowed && !bucketAllowed {
return ErrAccessDenied
@@ -580,7 +574,6 @@ func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Reques
TableBucketName: bucketName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
return ErrAccessDenied
}
@@ -917,7 +910,6 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
bucketAllowed := CheckPermissionWithContext("DeleteTable", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
@@ -926,7 +918,6 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
if !tableAllowed && !bucketAllowed {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table")
@@ -1062,7 +1053,6 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
bucketAllowed := CheckPermissionWithContext("UpdateTable", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
@@ -1071,7 +1061,6 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
if !tableAllowed && !bucketAllowed {
+1 -9
View File
@@ -20,10 +20,7 @@ type Manager struct {
// NewManager creates a new Manager.
func NewManager() *Manager {
m := &Manager{handler: NewS3TablesHandler()}
// Default to allowing access when IAM is not configured
m.handler.SetDefaultAllow(true)
return m
return &Manager{handler: NewS3TablesHandler()}
}
// SetRegion sets the AWS region for ARN generation.
@@ -36,11 +33,6 @@ func (m *Manager) SetAccountID(accountID string) {
m.handler.SetAccountID(accountID)
}
// SetDefaultAllow sets whether to allow access by default.
func (m *Manager) SetDefaultAllow(allow bool) {
m.handler.SetDefaultAllow(allow)
}
// Execute runs an S3 Tables operation and decodes the response into resp (if provided).
func (m *Manager) Execute(ctx context.Context, filerClient FilerClient, operation string, req interface{}, resp interface{}, identity string) error {
body, err := json.Marshal(req)

Some files were not shown because too many files have changed in this diff Show More