Compare commits

..
Author SHA1 Message Date
Chris LuandCopilot 8500a3bc56 ci: remove s3tests skip - test will be fixed in s3-tests repo
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 03:23:36 -08:00
Chris LuandCopilot 678c52e74f ci: skip s3tests test expecting incorrect delete behavior
Skip test_object_lock_delete_object_with_retention_and_marker in s3tests because it expects SeaweedFS incorrect behavior (allowing delete under COMPLIANCE retention). SeaweedFS now correctly implements AWS S3 behavior: delete of objects under COMPLIANCE retention returns AccessDenied and does not create a delete marker.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 03:20:05 -08:00
Chris LuandCopilot 8fac0cdab8 test: fix retention tests to expect correct AWS S3 behavior
Tests were expecting SeaweedFS old (incorrect) behavior where DeleteObject succeeded under COMPLIANCE retention. Updated to expect correct AWS S3 behavior: simple DELETE (without versionId) is blocked by active COMPLIANCE/legal-hold, returning AccessDenied instead of creating a delete marker.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 02:57:31 -08:00
Chris LuandCopilot 09bfb28db9 s3api: simplify objectLockVersionToCheckForDelete to single return value
Since all branches now return true (fail-closed), eliminate the boolean return value and the dead-code if guards. The function now clearly indicates that object lock checks always execute, with no opt-out path.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 02:08:32 -08:00
Chris LuandCopilot 3bd990d2ea s3api: address object lock delete review comments
- Remove versioningConfigured guard: object lock protections must apply to all buckets
- Combine identical switch cases for clarity
- Change default case to fail-closed (true) for safety
- Add test case for suspended versioning with specific versionId
- Update test expectations to reflect fail-closed default

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 02:00:30 -08:00
Chris LuandCopilot 5e040ba03d s3api: enforce object lock protections on delete (COMPLIANCE/GOVERNANCE)
Enforce object lock protections before creating delete markers; active COMPLIANCE retention denies deletes and active GOVERNANCE retention denies deletes unless bypass is authorized. Includes unit tests.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-02-16 01:52:08 -08:00
70 changed files with 340 additions and 1434 deletions
+1 -1
View File
@@ -111,7 +111,7 @@ jobs:
# Wait for S3 API
for i in {1..30}; do
if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
if curl -s http://localhost:8333/ > /dev/null 2>&1; then
echo "✓ S3 API is ready"
break
fi
-2
View File
@@ -461,8 +461,6 @@ jobs:
export S3_ENDPOINT="http://localhost:8006"
export S3_ACCESS_KEY="0555b35654ad1656d804"
export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
# Run the specific test that is equivalent to AWS S3 tagging behavior
make test-with-server || {
+1 -1
View File
@@ -97,7 +97,7 @@ jobs:
# Verify service accessibility
echo "=== Verifying Service Accessibility ==="
curl -f http://localhost:8080/realms/master
curl -s http://localhost:8333/healthz
curl -s http://localhost:8333
echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
# Run Keycloak-specific tests
@@ -35,7 +35,7 @@ jobs:
set -e
mkdir -p /tmp/data
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
- name: Setup Caddy
run: |
@@ -54,7 +54,7 @@ jobs:
- name: Start Caddy
run: |
./caddy start
until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
- name: Create Bucket
run: |
+5 -5
View File
@@ -46,7 +46,7 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/klauspost/compress v1.18.4
github.com/klauspost/compress v1.18.3
github.com/klauspost/reedsolomon v1.13.0
github.com/kurin/blazer v0.5.3
github.com/linxGnu/grocksdb v1.10.7
@@ -130,7 +130,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
github.com/cognusion/imaging v1.0.2
github.com/fluent/fluent-logger-golang v1.10.1
github.com/getsentry/sentry-go v0.42.0
github.com/getsentry/sentry-go v0.40.0
github.com/gin-contrib/sessions v1.0.4
github.com/gin-gonic/gin v1.11.0
github.com/go-ldap/ldap/v3 v3.4.12
@@ -142,7 +142,7 @@ require (
github.com/jhump/protoreflect v1.18.0
github.com/lib/pq v1.11.1
github.com/linkedin/goavro/v2 v2.14.1
github.com/mattn/go-sqlite3 v1.14.34
github.com/mattn/go-sqlite3 v1.14.33
github.com/minio/crc64nvme v1.1.1
github.com/orcaman/concurrent-map/v2 v2.0.1
github.com/parquet-go/parquet-go v0.26.4
@@ -150,7 +150,7 @@ require (
github.com/rabbitmq/amqp091-go v1.10.0
github.com/rclone/rclone v1.72.1
github.com/rdleal/intervalst v1.5.0
github.com/redis/go-redis/v9 v9.18.0
github.com/redis/go-redis/v9 v9.17.2
github.com/schollz/progressbar/v3 v3.19.0
github.com/seaweedfs/go-fuse/v2 v2.9.1
github.com/shirou/gopsutil/v4 v4.26.1
@@ -297,7 +297,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
github.com/Azure/go-ntlmssp v0.1.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
+10 -10
View File
@@ -569,8 +569,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -1053,8 +1053,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo=
github.com/getsentry/sentry-go v0.40.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
@@ -1487,8 +1487,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
@@ -1564,8 +1564,8 @@ github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhg
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
@@ -1814,8 +1814,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "4.13"
appVersion: "4.12"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.0.413
version: 4.0.412
+6
View File
@@ -19,6 +19,9 @@ services:
start_period: 10s
seaweedfs-volume:
build:
context: ../../../docker
dockerfile: Dockerfile.local
image: seaweedfs:local
container_name: seaweedfs-spark-volume
ports:
@@ -40,6 +43,9 @@ services:
start_period: 10s
seaweedfs-filer:
build:
context: ../../../docker
dockerfile: Dockerfile.local
image: seaweedfs:local
container_name: seaweedfs-spark-filer
ports:
+1 -1
View File
@@ -116,7 +116,7 @@
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>1.11.5</version>
<version>1.11.4</version>
</dependency>
<!-- Apache ZooKeeper - Fix CVEs -->
+4 -7
View File
@@ -12,7 +12,6 @@ FILER_PORT := 8888
TEST_TIMEOUT := 10m
TEST_PATTERN := TestCORS
SERVER_DIR := test-mini-data
S3_CONFIG := s3_test_config.json
# Default target
help:
@@ -81,15 +80,13 @@ start-server: check-deps
@echo "🔍 DEBUG: Creating volume directory..."
@mkdir -p $(SERVER_DIR)
@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
@echo "🔍 DEBUG: Command: AWS_ACCESS_KEY_ID=some_access_key1 AWS_SECRET_ACCESS_KEY=some_secret_key1 $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
@env AWS_ACCESS_KEY_ID=some_access_key1 \
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
$(WEED_BINARY) mini \
@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
@$(WEED_BINARY) mini \
-dir=$(SERVER_DIR) \
-s3.port=$(S3_PORT) \
-s3.config=$(S3_CONFIG) \
> weed-test.log 2>&1 & \
echo $$! > weed-server.pid
echo $$! > weed-test.pid
@echo "Waiting for S3 server to be ready..."
@for i in $$(seq 1 30); do \
@@ -100,7 +97,7 @@ start-server: check-deps
sleep 1; \
done; \
echo "S3 server failed to start"; \
exit 1
exit 1 > weed-server.pid
@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
@echo "🔍 DEBUG: Checking if PID is still running..."
@sleep 2
-27
View File
@@ -1,27 +0,0 @@
{
"identities": [
{
"name": "anonymous",
"actions": [
"Read",
"List"
]
},
{
"name": "admin",
"credentials": [
{
"accessKey": "some_access_key1",
"secretKey": "some_secret_key1"
}
],
"actions": [
"Admin",
"Read",
"List",
"Tagging",
"Write"
]
}
]
}
+5 -2
View File
@@ -1,6 +1,7 @@
package example
import (
"os"
"testing"
"time"
@@ -21,8 +22,10 @@ func TestIAMOperations(t *testing.T) {
// Set credentials before starting cluster
accessKey := "testkey123"
secretKey := "testsecret456"
t.Setenv("AWS_ACCESS_KEY_ID", accessKey)
t.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
os.Setenv("AWS_ACCESS_KEY_ID", accessKey)
os.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
defer os.Unsetenv("AWS_ACCESS_KEY_ID")
defer os.Unsetenv("AWS_SECRET_ACCESS_KEY")
// Create and start test cluster
cluster, err := startMiniCluster(t)
-8
View File
@@ -146,14 +146,6 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
return nil, fmt.Errorf("failed to create security.toml: %v", err)
}
// Set environment variables for admin credentials safely for this test
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
}
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
}
// Start weed mini in a goroutine by calling the command directly
cluster.wg.Add(1)
go func() {
-8
View File
@@ -216,14 +216,6 @@ enabled = true
err = os.WriteFile(credentialToml, []byte(credentialConfig), 0644)
require.NoError(t, err)
// Set environment variables for admin credentials safely for this test
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
}
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
}
cluster.wg.Add(1)
go func() {
defer cluster.wg.Done()
@@ -77,14 +77,14 @@ func TestObjectLockValidation(t *testing.T) {
require.NoError(t, err, "Setting Object Lock retention should succeed")
t.Log(" Object Lock retention applied successfully")
// Verify retention allows simple DELETE (creates delete marker) but blocks version deletion
// AWS S3 behavior: Simple DELETE (without version ID) is ALWAYS allowed and creates delete marker
// Verify retention blocks simple DELETE (COMPLIANCE mode is strict WORM)
// AWS S3 behavior: Simple DELETE (without version ID) is blocked by COMPLIANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker (AWS S3 behavior)")
t.Log(" Simple DELETE succeeded (creates delete marker - correct AWS behavior)")
require.Error(t, err, "Simple DELETE should be blocked by COMPLIANCE retention (AWS S3 behavior)")
t.Log(" Simple DELETE correctly blocked by COMPLIANCE retention")
// Now verify that DELETE with version ID is properly blocked by retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+7 -7
View File
@@ -329,12 +329,12 @@ func TestRetentionModeCompliance(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, types.ObjectLockRetentionModeCompliance, retentionResp.Retention.Mode)
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
// Try simple DELETE - should fail for COMPLIANCE mode (strict WORM)
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
require.Error(t, err, "Simple DELETE should be blocked by COMPLIANCE retention")
// Try DELETE with version ID - should fail for COMPLIANCE mode
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
@@ -388,12 +388,12 @@ func TestLegalHoldWorkflow(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, legalHoldResp.LegalHold.Status)
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
// Try simple DELETE - should fail due to legal hold
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
require.Error(t, err, "Simple DELETE should be blocked by legal hold")
// Try DELETE with version ID - should fail due to legal hold
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
@@ -591,12 +591,12 @@ func TestRetentionAndLegalHoldCombination(t *testing.T) {
})
require.NoError(t, err)
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
// Try simple DELETE - should fail due to legal hold + COMPLIANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
require.Error(t, err, "Simple DELETE should be blocked by legal hold")
// Try DELETE with version ID and bypass - should still fail due to legal hold
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
@@ -607,7 +607,7 @@ func TestRetentionAndLegalHoldCombination(t *testing.T) {
})
require.Error(t, err, "Legal hold should prevent deletion even with governance bypass")
// Remove legal hold (must specify version ID since latest version is now delete marker)
// Remove legal hold (must specify version ID)
_, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
@@ -42,12 +42,12 @@ func TestWORMRetentionIntegration(t *testing.T) {
})
require.NoError(t, err)
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
// Try simple DELETE - should fail due to GOVERNANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
require.Error(t, err, "Simple DELETE should be blocked by GOVERNANCE retention")
// Try DELETE with version ID - should fail due to GOVERNANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
@@ -325,12 +325,12 @@ func TestRetentionWithMultipartUpload(t *testing.T) {
})
require.NoError(t, err)
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
// Try simple DELETE - should fail due to GOVERNANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
})
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
require.Error(t, err, "Simple DELETE should be blocked by GOVERNANCE retention")
// Try DELETE with version ID - should fail due to GOVERNANCE retention
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
@@ -511,10 +511,8 @@ func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
t.Logf("Create table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
body, _ := io.ReadAll(resp.Body)
t.Fatalf("Failed to create table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
}
t.Logf("Created table bucket %s", bucketName)
+2 -66
View File
@@ -56,6 +56,8 @@ func TestPyIcebergRestCatalog(t *testing.T) {
cmd := exec.Command("docker", "run", "--rm",
"--add-host", "host.docker.internal:host-gateway",
"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", "test"),
"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", "test"),
"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
"-v", fmt.Sprintf("%s:/app:ro", testDir),
"iceberg-rest-test",
@@ -76,69 +78,3 @@ func TestPyIcebergRestCatalog(t *testing.T) {
t.Errorf("PyIceberg test failed: %v", err)
}
}
// TestPyIcebergRestCatalogAuthenticated tests the Iceberg REST Catalog using PyIceberg with authentication.
// This test uses the default admin credentials that SeaweedFS creates on startup.
func TestPyIcebergRestCatalogAuthenticated(t *testing.T) {
if testing.Short() {
t.Skip("Skipping integration test in short mode")
}
env := NewTestEnvironment(t)
defer env.Cleanup(t)
if !env.dockerAvailable {
t.Skip("Docker not available, skipping PyIceberg integration test")
}
// Use default admin credentials
testAccessKey := "admin"
testSecretKey := "admin"
// Start SeaweedFS (it will use default admin credentials from environment if set)
env.StartSeaweedFS(t)
// Create the test bucket first (using unauthenticated request, which works with DefaultAllow)
bucketName := "pyiceberg-auth-test"
createTableBucket(t, env, bucketName)
// Build the test working directory path
testDir := filepath.Join(env.seaweedDir, "test", "s3tables", "catalog")
// Run PyIceberg test using Docker with authentication
catalogURL := fmt.Sprintf("http://host.docker.internal:%d", env.icebergPort)
s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
warehouse := fmt.Sprintf("s3://%s/", bucketName)
// Build the test image first for faster repeated runs
buildCmd := exec.Command("docker", "build", "-t", "iceberg-rest-test", "-f", "Dockerfile.pyiceberg", ".")
buildCmd.Dir = testDir
if out, err := buildCmd.CombinedOutput(); err != nil {
t.Fatalf("Failed to build test image: %v\n%s", err, string(out))
}
cmd := exec.Command("docker", "run", "--rm",
"--add-host", "host.docker.internal:host-gateway",
"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
"-v", fmt.Sprintf("%s:/app:ro", testDir),
"iceberg-rest-test",
"python3", "/app/test_rest_catalog_auth.py",
"--catalog-url", catalogURL,
"--warehouse", warehouse,
"--prefix", bucketName,
"--access-key", testAccessKey,
"--secret-key", testSecretKey,
)
cmd.Dir = testDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
t.Logf("Running PyIceberg REST catalog test with authentication...")
t.Logf(" Catalog URL: %s", catalogURL)
t.Logf(" Warehouse: %s", warehouse)
t.Logf(" Access Key: %s", testAccessKey)
if err := cmd.Run(); err != nil {
t.Errorf("PyIceberg authenticated test failed: %v", err)
}
}
@@ -1,36 +0,0 @@
package catalog
import (
"fmt"
"io"
"net/http"
"testing"
)
// verifyTableBucketMetadata verifies that a table bucket was created with proper metadata
func verifyTableBucketMetadata(t *testing.T, env *TestEnvironment, bucketName string) {
t.Helper()
// Use S3Tables REST API to get the bucket
endpoint := fmt.Sprintf("http://localhost:%d/buckets/%s", env.s3Port, bucketName)
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
if err != nil {
t.Fatalf("Failed to create request: %v", err)
}
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Failed to get table bucket %s: %v", bucketName, err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
t.Logf("Get table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
if resp.StatusCode != http.StatusOK {
t.Fatalf("Failed to get table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
}
t.Logf("Verified table bucket %s exists with metadata", bucketName)
}
@@ -201,7 +201,6 @@ def main():
"uri": args.catalog_url,
"warehouse": args.warehouse,
"prefix": args.prefix,
"s3.anonymous": "true", # Disable AWS request signing for unauthenticated access
}
)
print(f"Successfully connected to catalog on attempt {attempt + 1}")
@@ -1,223 +0,0 @@
#!/usr/bin/env python3
"""
Iceberg REST Catalog Compatibility Test for SeaweedFS (Authenticated)
This script tests the Iceberg REST Catalog API compatibility with authentication.
Usage:
python3 test_rest_catalog_auth.py --catalog-url http://localhost:8182 \\
--access-key admin --secret-key admin
Requirements:
pip install pyiceberg[s3fs]
"""
import argparse
import sys
from pyiceberg.catalog import load_catalog
from pyiceberg.schema import Schema
from pyiceberg.types import (
IntegerType,
LongType,
StringType,
NestedField,
)
from pyiceberg.exceptions import (
NamespaceAlreadyExistsError,
NoSuchNamespaceError,
TableAlreadyExistsError,
NoSuchTableError,
)
def test_config_endpoint(catalog):
"""Test that the catalog config endpoint returns valid configuration."""
print("Testing /v1/config endpoint...")
# The catalog is already loaded which means config endpoint worked
print(" /v1/config endpoint working")
return True
def test_namespace_operations(catalog, prefix):
"""Test namespace CRUD operations."""
print("Testing namespace operations...")
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
# List initial namespaces
namespaces = catalog.list_namespaces()
print(f" Initial namespaces: {namespaces}")
# Create namespace
try:
catalog.create_namespace(namespace)
print(f" Created namespace: {namespace}")
except NamespaceAlreadyExistsError:
print(f" ! Namespace already exists: {namespace}")
# List namespaces (should include our new one)
namespaces = catalog.list_namespaces()
if namespace in namespaces:
print(" Namespace appears in list")
else:
print(f" Namespace not found in list: {namespaces}")
return False
# Get namespace properties
try:
props = catalog.load_namespace_properties(namespace)
print(f" Loaded namespace properties: {props}")
except NoSuchNamespaceError:
print(f" Failed to load namespace properties")
return False
return True
def test_table_operations(catalog, prefix):
"""Test table CRUD operations."""
print("Testing table operations...")
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
table_name = "auth_test_table"
table_id = namespace + (table_name,)
# Define a simple schema
schema = Schema(
NestedField(field_id=1, name="id", field_type=LongType(), required=True),
NestedField(field_id=2, name="name", field_type=StringType(), required=False),
NestedField(field_id=3, name="age", field_type=IntegerType(), required=False),
)
# Create table
try:
table = catalog.create_table(
identifier=table_id,
schema=schema,
)
print(f" Created table: {table_id}")
except TableAlreadyExistsError:
print(f" ! Table already exists: {table_id}")
_ = catalog.load_table(table_id)
# List tables
tables = catalog.list_tables(namespace)
if table_name in [t[1] for t in tables]:
print(" Table appears in list")
else:
print(f" Table not found in list: {tables}")
return False
# Load table
try:
loaded_table = catalog.load_table(table_id)
print(f" Loaded table: {loaded_table.name()}")
print(f" Schema: {loaded_table.schema()}")
print(f" Location: {loaded_table.location()}")
except NoSuchTableError:
print(f" Failed to load table")
return False
return True
def test_cleanup(catalog, prefix):
"""Test table and namespace deletion."""
print("Testing cleanup operations...")
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
table_id = namespace + ("auth_test_table",)
# Drop table
try:
catalog.drop_table(table_id)
print(f" Dropped table: {table_id}")
except NoSuchTableError:
print(f" ! Table already deleted: {table_id}")
# Drop namespace
try:
catalog.drop_namespace(namespace)
print(f" Dropped namespace: {namespace}")
except NoSuchNamespaceError:
print(f" ! Namespace already deleted: {namespace}")
except Exception as e:
print(f" ? Namespace drop error (may be expected): {e}")
return True
def main():
parser = argparse.ArgumentParser(description="Test Iceberg REST Catalog with authentication")
parser.add_argument("--catalog-url", required=True, help="Iceberg REST Catalog URL")
parser.add_argument("--warehouse", default="s3://iceberg-test/", help="Warehouse location")
parser.add_argument("--prefix", required=True, help="Table bucket prefix")
parser.add_argument("--access-key", required=True, help="AWS Access Key ID")
parser.add_argument("--secret-key", required=True, help="AWS Secret Access Key")
parser.add_argument("--skip-cleanup", action="store_true", help="Skip cleanup at the end")
args = parser.parse_args()
print(f"Connecting to Iceberg REST Catalog at: {args.catalog_url}")
print(f"Warehouse: {args.warehouse}")
print(f"Prefix: {args.prefix}")
print(f"Using authenticated access with key: {args.access_key}")
print()
# Load the REST catalog with authentication
import time
max_retries = 10
catalog = None
for attempt in range(max_retries):
try:
catalog = load_catalog(
"rest",
**{
"type": "rest",
"uri": args.catalog_url,
"warehouse": args.warehouse,
"prefix": args.prefix,
"s3.access-key-id": args.access_key,
"s3.secret-access-key": args.secret_key,
}
)
print(f"Successfully connected to catalog on attempt {attempt + 1}")
break
except Exception as e:
if attempt < max_retries - 1:
print(f" Attempt {attempt + 1} failed, retrying in 2s... ({e})")
time.sleep(2)
else:
print(f" All {max_retries} attempts failed.")
raise e
# Run tests
tests = [
("Config Endpoint", lambda: test_config_endpoint(catalog)),
("Namespace Operations", lambda: test_namespace_operations(catalog, args.prefix)),
("Table Operations", lambda: test_table_operations(catalog, args.prefix)),
]
if not args.skip_cleanup:
tests.append(("Cleanup", lambda: test_cleanup(catalog, args.prefix)))
passed = 0
failed = 0
for name, test_fn in tests:
print(f"\n{'='*50}")
try:
if test_fn():
passed += 1
print(f"PASSED: {name}")
else:
failed += 1
print(f"FAILED: {name}")
except Exception as e:
failed += 1
print(f"ERROR in {name}: {e}")
print(f"\n{'='*50}")
print(f"Results: {passed} passed, {failed} failed")
return 0 if failed == 0 else 1
if __name__ == "__main__":
sys.exit(main())
@@ -556,14 +556,6 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
return nil, fmt.Errorf("failed to create security.toml: %v", err)
}
// Set environment variables for admin credentials safely for this test
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
}
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
}
// Start weed mini in a goroutine by calling the command directly
cluster.wg.Add(1)
go func() {
@@ -1,14 +0,0 @@
package maintenance
const DefaultMasterMaintenanceScripts = `
lock
ec.encode -fullPercent=95 -quietFor=1h
ec.rebuild -apply
ec.balance -apply
fs.log.purge -daysAgo=7
volume.deleteEmpty -quietFor=24h -apply
volume.balance -apply
volume.fix.replication -apply
s3.clean.uploads -timeAgo=24h
unlock
`
+1 -1
View File
@@ -137,7 +137,7 @@ func backupFromLocation(volumeServer pb.ServerAddress, grpcDialOption grpc.DialO
// Handle compaction if needed
if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
if err = v.CompactByIndex(nil); err != nil {
if err = v.Compact2(0, 0, nil); err != nil {
v.Close()
return fmt.Errorf("compacting volume: %w", err), false
}
+9 -20
View File
@@ -1,8 +1,6 @@
package command
import (
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
@@ -20,9 +18,8 @@ var cmdCompact = &Command{
The compacted .dat file is stored as .cpd file.
The compacted .idx file is stored as .cpx file.
Supports two compaction methods:
* data: compacts based on the .dat file, works if .idx file is corrupted.
* index: compacts based on the .idx file, works if deletion happened but not written to .dat files.
For method=0, it compacts based on the .dat file, works if .idx file is corrupted.
For method=1, it compacts based on the .idx file, works if deletion happened but not written to .dat files.
`,
}
@@ -31,7 +28,7 @@ var (
compactVolumePath = cmdCompact.Flag.String("dir", ".", "data directory to store files")
compactVolumeCollection = cmdCompact.Flag.String("collection", "", "volume collection name")
compactVolumeId = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
compactMethod = cmdCompact.Flag.String("method", "data", "option to choose which compact method (data/index)")
compactMethod = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 (default) or 1.")
compactVolumePreallocate = cmdCompact.Flag.Int64("preallocateMB", 0, "preallocate volume disk space")
)
@@ -41,29 +38,21 @@ func runCompact(cmd *Command, args []string) bool {
return false
}
preallocateBytes := *compactVolumePreallocate * (1 << 20)
preallocate := *compactVolumePreallocate * (1 << 20)
vid := needle.VolumeId(*compactVolumeId)
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocateBytes, needle.GetCurrentVersion(), 0, 0)
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, needle.GetCurrentVersion(), 0, 0)
if err != nil {
glog.Fatalf("Load Volume [ERROR] %s\n", err)
}
opts := &storage.CompactOptions{
PreallocateBytes: preallocateBytes,
MaxBytesPerSecond: 0, // unlimited
}
switch strings.ToLower(*compactMethod) {
case "data":
if err = v.CompactByVolumeData(opts); err != nil {
if *compactMethod == 0 {
if err = v.Compact(preallocate, 0); err != nil {
glog.Fatalf("Compact Volume [ERROR] %s\n", err)
}
case "index":
if err = v.CompactByIndex(opts); err != nil {
} else {
if err = v.Compact2(preallocate, 0, nil); err != nil {
glog.Fatalf("Compact Volume [ERROR] %s\n", err)
}
default:
glog.Fatalf("unsupported compaction method %q", *compactMethod)
}
return true
+4 -14
View File
@@ -355,9 +355,6 @@ func isFlagPassed(name string) bool {
// isPortOpenOnIP checks if a port is available for binding on a specific IP address
func isPortOpenOnIP(ip string, port int) bool {
if port <= 0 || port > 65535 {
return false
}
listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", ip, port))
if err != nil {
return false
@@ -369,9 +366,6 @@ func isPortOpenOnIP(ip string, port int) bool {
// isPortAvailable checks if a port is available on any interface
// This is more comprehensive than checking a single IP
func isPortAvailable(port int) bool {
if port <= 0 || port > 65535 {
return false
}
// Try to listen on all interfaces (0.0.0.0)
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
@@ -387,10 +381,6 @@ func isPortAvailable(port int) bool {
func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPorts map[int]bool) int {
for i := 0; i < maxAttempts; i++ {
port := startPort + i
if port > 65535 {
// Wrap around to a lower range if we exceed 65535
port = 10000 + (port % 65535)
}
// Skip ports reserved for gRPC calculation
if reservedPorts[port] {
continue
@@ -408,15 +398,15 @@ func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPo
// If the port is not available, it finds the next available port and updates the pointer
// The reservedPorts map contains ports that should not be allocated (for gRPC collision avoidance)
func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reservedPorts map[int]bool, flagName string) error {
// Check if this port was explicitly specified by the user (from CLI, before config file was applied)
isExplicitPort := explicitPortFlags[flagName]
if *portPtr == 0 {
if portPtr == nil {
return nil
}
original := *portPtr
// Check if this port was explicitly specified by the user (from CLI, before config file was applied)
isExplicitPort := explicitPortFlags[flagName]
// Skip if this port is reserved for gRPC calculation
if reservedPorts[original] {
if isExplicitPort {
+1 -11
View File
@@ -1,15 +1,6 @@
package scaffold
import (
_ "embed"
"strings"
"github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
)
func init() {
Master = strings.ReplaceAll(masterTemplate, "{{DEFAULT_MAINTENANCE_SCRIPTS}}", maintenance.DefaultMasterMaintenanceScripts)
}
import _ "embed"
//go:embed filer.toml
var Filer string
@@ -24,7 +15,6 @@ var Replication string
var Security string
//go:embed master.toml
var masterTemplate string
var Master string
//go:embed shell.toml
+11 -1
View File
@@ -6,7 +6,17 @@
[master.maintenance]
# periodically run these scripts are the same as running them from 'weed shell'
scripts = """{{DEFAULT_MAINTENANCE_SCRIPTS}}"""
scripts = """
lock
ec.encode -fullPercent=95 -quietFor=1h
ec.rebuild -apply
ec.balance -apply
volume.deleteEmpty -quietFor=24h -apply
volume.balance -apply
volume.fix.replication -apply
s3.clean.uploads -timeAgo=24h
unlock
"""
sleep_minutes = 17 # sleep minutes between each script execution
-1
View File
@@ -43,7 +43,6 @@ expires_after_seconds = 10 # seconds
# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
# - f.e. the S3 API Shim generates the JWT
# - the Filer server validates the JWT on writing
# NOTE: This key is ALSO used as a fallback signing key for S3 STS if s3.iam.config does not specify a signingKey.
# the jwt defaults to expire after 10 seconds.
[jwt.filer_signing]
key = ""
-8
View File
@@ -654,14 +654,6 @@ func (m *IAMManager) GetSTSService() *sts.STSService {
return m.stsService
}
// DefaultAllow returns whether the default effect is Allow
func (m *IAMManager) DefaultAllow() bool {
if !m.initialized || m.policyEngine == nil {
return true // Default to true if not initialized
}
return m.policyEngine.DefaultAllow()
}
// parseJWTTokenForTrustPolicy parses a JWT token to extract claims for trust policy evaluation
func parseJWTTokenForTrustPolicy(tokenString string) (map[string]interface{}, error) {
// Simple JWT parsing without verification (for trust policy context only)
-8
View File
@@ -324,14 +324,6 @@ func (e *PolicyEngine) IsInitialized() bool {
return e.initialized
}
// DefaultAllow returns whether the default effect is Allow
func (e *PolicyEngine) DefaultAllow() bool {
if e.config == nil {
return true // Default to Allow if not configured
}
return e.config.DefaultEffect == string(EffectAllow)
}
// AddPolicy adds a policy to the engine (filerAddress ignored for memory stores)
func (e *PolicyEngine) AddPolicy(filerAddress string, name string, policy *PolicyDocument) error {
if !e.initialized {
-18
View File
@@ -270,9 +270,6 @@ func (s *STSService) Initialize(config *STSConfig) error {
return fmt.Errorf(ErrConfigCannotBeNil)
}
// Apply defaults before validation
config.ApplyDefaults()
if err := s.validateConfig(config); err != nil {
return fmt.Errorf("invalid STS configuration: %w", err)
}
@@ -291,21 +288,6 @@ func (s *STSService) Initialize(config *STSConfig) error {
return nil
}
// ApplyDefaults applies default values to the STS configuration
func (c *STSConfig) ApplyDefaults() {
if c.TokenDuration.Duration <= 0 {
c.TokenDuration.Duration = time.Duration(DefaultTokenDuration) * time.Second
}
if c.MaxSessionLength.Duration <= 0 {
c.MaxSessionLength.Duration = time.Duration(DefaultMaxSessionLength) * time.Second
}
if c.Issuer == "" {
c.Issuer = DefaultIssuer
}
}
// validateConfig validates the STS configuration
func (s *STSService) validateConfig(config *STSConfig) error {
if config.TokenDuration.Duration <= 0 {
-23
View File
@@ -75,34 +75,11 @@ func TestSTSServiceInitialization(t *testing.T) {
} else {
assert.NoError(t, err)
assert.True(t, service.IsInitialized())
// Verify defaults if applicable
if tt.config.Issuer == "" {
assert.Equal(t, DefaultIssuer, service.Config.Issuer)
}
if tt.config.TokenDuration.Duration == 0 {
assert.Equal(t, time.Duration(DefaultTokenDuration)*time.Second, service.Config.TokenDuration.Duration)
}
}
})
}
}
func TestSTSServiceDefaults(t *testing.T) {
service := NewSTSService()
config := &STSConfig{
SigningKey: []byte("test-signing-key"),
// Missing duration and issuer
}
err := service.Initialize(config)
assert.NoError(t, err)
assert.Equal(t, DefaultIssuer, config.Issuer)
assert.Equal(t, time.Duration(DefaultTokenDuration)*time.Second, config.TokenDuration.Duration)
assert.Equal(t, time.Duration(DefaultMaxSessionLength)*time.Second, config.MaxSessionLength.Duration)
}
// TestAssumeRoleWithWebIdentity tests role assumption with OIDC tokens
func TestAssumeRoleWithWebIdentity(t *testing.T) {
service := setupTestSTSService(t)
+1 -10
View File
@@ -89,9 +89,7 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli
GrpcDialOption: option.GrpcDialOption,
}
// Initialize FilerClient for IAM - explicit filers only (no discovery as FilerGroup unspecified)
filerClient := wdclient.NewFilerClient(option.Filers, option.GrpcDialOption, "")
iam := s3api.NewIdentityAccessManagementWithStore(&s3Option, filerClient, explicitStore)
iam := s3api.NewIdentityAccessManagementWithStore(&s3Option, explicitStore)
configure.credentialManager = iam.GetCredentialManager()
iamApiServer = &IamApiServer{
@@ -102,13 +100,6 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli
masterClient: masterClient,
}
// Keep attempting to load configuration from filer now that we have a client
go func() {
if err := iam.LoadS3ApiConfigurationFromCredentialManager(); err != nil {
glog.Warningf("Failed to load IAM config from credential manager after client update: %v", err)
}
}()
iamApiServer.registerRouter(router)
return iamApiServer, nil
+10 -69
View File
@@ -22,7 +22,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
// Import KMS providers to register them
_ "github.com/seaweedfs/seaweedfs/weed/kms/aws"
@@ -55,7 +54,7 @@ type IdentityAccessManagement struct {
domain string
isAuthEnabled bool
credentialManager *credential.CredentialManager
filerClient *wdclient.FilerClient
filerClient filer_pb.SeaweedFilerClient
grpcDialOption grpc.DialOption
// IAM Integration for advanced features
@@ -133,37 +132,15 @@ func (c *Credential) isCredentialExpired() bool {
return c.Expiration > 0 && c.Expiration < time.Now().Unix()
}
// NewIdentityAccessManagement creates a new IAM manager
// SetFilerClient updates the filer client and its associated credential store
func (iam *IdentityAccessManagement) SetFilerClient(filerClient *wdclient.FilerClient) {
iam.m.Lock()
iam.filerClient = filerClient
iam.m.Unlock()
if iam.credentialManager == nil || filerClient == nil {
return
}
// Update credential store to use FilerClient's current filer for HA
if store := iam.credentialManager.GetStore(); store != nil {
if filerFuncSetter, ok := store.(interface {
SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption)
}); ok {
filerFuncSetter.SetFilerAddressFunc(filerClient.GetCurrentFiler, iam.grpcDialOption)
}
}
func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement {
return NewIdentityAccessManagementWithStore(option, "")
}
func NewIdentityAccessManagement(option *S3ApiServerOption, filerClient *wdclient.FilerClient) *IdentityAccessManagement {
return NewIdentityAccessManagementWithStore(option, filerClient, "")
}
func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient *wdclient.FilerClient, explicitStore string) *IdentityAccessManagement {
func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitStore string) *IdentityAccessManagement {
iam := &IdentityAccessManagement{
domain: option.DomainName,
hashes: make(map[string]*sync.Pool),
hashCounters: make(map[string]*int32),
filerClient: filerClient,
}
// Always initialize credential manager with fallback to defaults
@@ -195,25 +172,6 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient
iam.credentialManager = credentialManager
iam.stopChan = make(chan struct{})
iam.grpcDialOption = option.GrpcDialOption
// Initialize default anonymous identity
// This ensures consistent behavior for anonymous access:
// 1. In simple auth mode (no IAM integration):
// - lookupAnonymous returns this identity
// - VerifyActionPermission checks actions (which are empty) -> Denies access
// - This preserves the secure-by-default behavior for simple auth
// 2. In advanced IAM mode (with Policy Engine):
// - lookupAnonymous returns this identity
// - VerifyActionPermission proceeds to Policy Engine
// - Policy Engine evaluates against policies (DefaultEffect=Allow if no config)
// - This enables the flexible "Open by Default" for zero-config startup
iam.identityAnonymous = &Identity{
Name: "anonymous",
Account: &AccountAnonymous,
Actions: []Action{},
IsStatic: true,
}
// First, try to load configurations from file or filer
startConfigFile := option.Config
@@ -594,16 +552,6 @@ func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3
}
}
// Ensure anonymous identity exists
if identityAnonymous == nil {
identityAnonymous = &Identity{
Name: "anonymous",
Account: accounts[AccountAnonymous.Id],
Actions: []Action{},
IsStatic: true,
}
}
// atomically switch
iam.identities = identities
iam.identityAnonymous = identityAnonymous
@@ -624,9 +572,6 @@ func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3
}
}
if !exists {
if len(envIdent.Credentials) == 0 {
continue
}
iam.identities = append(iam.identities, envIdent)
iam.accessKeyIdent[envIdent.Credentials[0].AccessKey] = envIdent
iam.nameToIdentity[envIdent.Name] = envIdent
@@ -1047,8 +992,7 @@ func (iam *IdentityAccessManagement) LookupByAccessKey(accessKey string) (identi
return iam.lookupByAccessKey(accessKey)
}
// LookupAnonymous returns the anonymous identity if it exists
func (iam *IdentityAccessManagement) LookupAnonymous() (identity *Identity, found bool) {
func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) {
iam.m.RLock()
defer iam.m.RUnlock()
if iam.identityAnonymous != nil {
@@ -1168,9 +1112,6 @@ func (iam *IdentityAccessManagement) handleAuthResult(w http.ResponseWriter, r *
// Wrapper to maintain backward compatibility
func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) {
identity, err, _ := iam.authRequestWithAuthType(r, action)
if err != s3err.ErrNone {
return nil, err
}
return identity, err
}
@@ -1232,7 +1173,7 @@ func (iam *IdentityAccessManagement) authenticateRequestInternal(r *http.Request
}
case authTypeAnonymous:
amzAuthType = "Anonymous"
if identity, found = iam.LookupAnonymous(); !found {
if identity, found = iam.lookupAnonymous(); !found {
r.Header.Set(s3_constants.AmzAuthType, amzAuthType)
return identity, s3err.ErrAccessDenied, reqAuthType
}
@@ -1271,8 +1212,8 @@ func (iam *IdentityAccessManagement) authRequestWithAuthType(r *http.Request, ac
// through buckets and checking permissions for each. Skip the global check here.
policyAllows := false
if action == s3_constants.ACTION_LIST && bucket == "" && identity.Name != s3_constants.AccountAnonymousId {
// ListBuckets operation for authenticated users - authorization handled per-bucket in the handler
if action == s3_constants.ACTION_LIST && bucket == "" {
// ListBuckets operation - authorization handled per-bucket in the handler
} else {
// First check bucket policy if one exists
// Bucket policies can grant or deny access to specific users/principals
@@ -1366,8 +1307,8 @@ func (iam *IdentityAccessManagement) AuthSignatureOnly(r *http.Request) (*Identi
return identity, s3err.ErrNotImplemented
}
case authTypeAnonymous:
// Anonymous users can be authenticated, but authorization is handled separately
return iam.identityAnonymous, s3err.ErrNone
// Anonymous users cannot use IAM API
return identity, s3err.ErrAccessDenied
default:
return identity, s3err.ErrNotImplemented
}
+3 -3
View File
@@ -450,7 +450,7 @@ func TestNewIdentityAccessManagementWithStoreEnvVars(t *testing.T) {
option := &S3ApiServerOption{
Config: "", // No config file - this should trigger environment variable fallback
}
iam := NewIdentityAccessManagementWithStore(option, nil, string(credential.StoreTypeMemory))
iam := NewIdentityAccessManagementWithStore(option, string(credential.StoreTypeMemory))
if tt.expectEnvIdentity {
// Should have exactly one identity from environment variables
@@ -510,7 +510,7 @@ func TestConfigFileWithNoIdentitiesAllowsEnvVars(t *testing.T) {
option := &S3ApiServerOption{
Config: tmpFile.Name(),
}
iam := NewIdentityAccessManagementWithStore(option, nil, string(credential.StoreTypeMemory))
iam := NewIdentityAccessManagementWithStore(option, string(credential.StoreTypeMemory))
// Should have exactly one identity from environment variables
assert.Len(t, iam.identities, 1, "Should have exactly one identity from environment variables even when config file exists with no identities")
@@ -762,7 +762,7 @@ func TestSignatureVerificationDoesNotCheckPermissions(t *testing.T) {
}
func TestStaticIdentityProtection(t *testing.T) {
iam := NewIdentityAccessManagement(&S3ApiServerOption{}, nil)
iam := NewIdentityAccessManagement(&S3ApiServerOption{})
// Add a static identity
staticIdent := &Identity{
+1 -1
View File
@@ -66,7 +66,7 @@ func TestReproIssue7912(t *testing.T) {
option := &S3ApiServerOption{
Config: tmpFile.Name(),
}
iam := NewIdentityAccessManagementWithStore(option, nil, "memory")
iam := NewIdentityAccessManagementWithStore(option, "memory")
assert.True(t, iam.isEnabled(), "Auth should be enabled")
-4
View File
@@ -44,10 +44,6 @@ func (m *MockIAMIntegration) ValidateTrustPolicyForPrincipal(ctx context.Context
return nil
}
func (m *MockIAMIntegration) DefaultAllow() bool {
return true
}
// TestVerifyV4SignatureWithSTSIdentity tests that verifyV4Signature properly handles STS identities
// by falling back to IAM authorization when shouldCheckPermissions is true
func TestVerifyV4SignatureWithSTSIdentity(t *testing.T) {
+3 -3
View File
@@ -22,7 +22,7 @@ func TestSTSIdentityPolicyNamesPopulation(t *testing.T) {
stsService, config := setupTestSTSService(t)
// Create IAM with STS integration
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, "memory")
s3iam := &S3IAMIntegration{
stsService: stsService,
}
@@ -264,7 +264,7 @@ func TestValidateSTSSessionTokenIntegration(t *testing.T) {
stsService, config := setupTestSTSService(t)
// Create IAM with STS integration
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, "memory")
s3iam := &S3IAMIntegration{
stsService: stsService,
}
@@ -311,7 +311,7 @@ func TestSTSIdentityClaimsPopulation(t *testing.T) {
stsService, config := setupTestSTSService(t)
// Create IAM with STS integration
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, "memory")
s3iam := &S3IAMIntegration{
stsService: stsService,
}
-156
View File
@@ -1,156 +0,0 @@
package s3api
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadIAMManagerFromConfig_Defaults(t *testing.T) {
// Create a temporary config file with minimal content (just policy)
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "iam_config.json")
configContent := `{
"sts": {
"providers": []
},
"policy": {
"storeType": "memory",
"defaultEffect": "Allow"
}
}`
err := os.WriteFile(configPath, []byte(configContent), 0644)
assert.NoError(t, err)
// dummy filer address provider
filerProvider := func() string { return "localhost:8888" }
defaultSigningKeyProvider := func() string { return "default-secure-signing-key" }
// Load the manager
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
assert.NoError(t, err)
assert.NotNil(t, manager)
}
func TestLoadIAMManagerFromConfig_Overrides(t *testing.T) {
// Create a temporary config file with EXPLICIT values
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "iam_config_explicit.json")
configContent := `{
"sts": {
"tokenDuration": "2h",
"maxSessionLength": "24h",
"issuer": "custom-issuer",
"signingKey": "ZXhwbGljaXQtc2lnbmluZy1rZXktMTIzNDU="
},
"policy": {
"storeType": "memory",
"defaultEffect": "Allow"
}
}`
// Base64 encoded "explicit-signing-key-12345" is "ZXhwbGljaXQtc2lnbmluZy1rZXktMTIzNDU="
err := os.WriteFile(configPath, []byte(configContent), 0644)
assert.NoError(t, err)
filerProvider := func() string { return "localhost:8888" }
defaultSigningKeyProvider := func() string { return "default-secure-signing-key" }
// Load
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
assert.NoError(t, err)
assert.NotNil(t, manager)
}
func TestLoadIAMManagerFromConfig_PartialDefaults(t *testing.T) {
// Test that partial configs (e.g. providing SigningKey but not Duration) work
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "iam_config_partial.json")
// Signing key provided in JSON, others missing
configContent := `{
"sts": {
"signingKey": "anNvbi1wcm92aWRlZC1rZXktMTIzNDU="
},
"policy": {
"storeType": "memory",
"defaultEffect": "Allow"
}
}`
err := os.WriteFile(configPath, []byte(configContent), 0644)
assert.NoError(t, err)
filerProvider := func() string { return "localhost:8888" }
// Default signing key provided but should be IGNORED because JSON has one
defaultSigningKeyProvider := func() string { return "server-default-key-should-be-ignored" }
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
assert.NoError(t, err)
assert.NotNil(t, manager)
}
func TestLoadIAMManagerFromConfig_ExplicitEmptyKey(t *testing.T) {
// Test that if JSON has empty signing key string, it still falls back
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "iam_config_empty_key.json")
// Signing key explicitly empty
configContent := `{
"sts": {
"signingKey": ""
},
"policy": {
"storeType": "memory",
"defaultEffect": "Allow"
}
}`
err := os.WriteFile(configPath, []byte(configContent), 0644)
assert.NoError(t, err)
filerProvider := func() string { return "localhost:8888" }
defaultSigningKeyProvider := func() string { return "fallback-key-should-be-used" }
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
assert.NoError(t, err)
assert.NotNil(t, manager)
}
func TestLoadIAMManagerFromConfig_MissingKeyError(t *testing.T) {
// Test that if BOTH keys are empty, it fails with a clear error
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "iam_config_all_empty.json")
// Signing key explicitly empty in JSON
configContent := `{
"sts": {
"signingKey": ""
},
"policy": {
"storeType": "memory",
"defaultEffect": "Allow"
}
}`
err := os.WriteFile(configPath, []byte(configContent), 0644)
assert.NoError(t, err)
filerProvider := func() string { return "localhost:8888" }
defaultSigningKeyProvider := func() string { return "" } // Empty default too
// Ensure no SSE-S3 key interferes (global state in tests is tricky, but let's assume clean state or no mock)
// Ideally we would mock GetSSES3KeyManager().GetMasterKey() but it's a global singleton.
// For this unit test, if the global key manager has no key, it should fail.
_, err = loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
// Should return a clear error
assert.Error(t, err)
assert.Contains(t, err.Error(), "no signing key found for STS service")
}
-50
View File
@@ -1,50 +0,0 @@
package s3api
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLoadIAMManagerFromConfig_OptionalConfig(t *testing.T) {
// Mock dependencies
filerAddressProvider := func() string { return "localhost:8888" }
getFilerSigningKey := func() string { return "test-signing-key" }
// Test Case 1: Empty config path should load defaults
iamManager, err := loadIAMManagerFromConfig("", filerAddressProvider, getFilerSigningKey)
require.NoError(t, err)
require.NotNil(t, iamManager)
// Verify STS Service is initialized with defaults
stsService := iamManager.GetSTSService()
assert.NotNil(t, stsService)
// Verify defaults are applied
// Since we can't easily access the internal config of stsService,
// we rely on the fact that initialization succeeded without error.
// We can also verify that the policy engine uses memory store by default.
// Verify Policy Engine is initialized with defaults (Memory store, Deny effect)
// Again, internal state might be hard to access directly, but successful init implies defaults worked.
}
func TestLoadIAMManagerFromConfig_EmptyConfigWithFallbackKey(t *testing.T) {
// Mock dependencies where getFilerSigningKey returns empty, forcing fallback logic
// Initialize IAM with empty config (should trigger defaults)
// We pass empty string for config file path
option := &S3ApiServerOption{
Config: "",
IamConfig: "",
EnableIam: true,
}
iamManager := NewIdentityAccessManagementWithStore(option, nil, "memory")
// Verify identityAnonymous is initialized
// This confirms the fix for anonymous access in zero-config mode
anonIdentity, found := iamManager.LookupAnonymous()
assert.True(t, found, "Anonymous identity should be found by default")
assert.NotNil(t, anonIdentity, "Anonymous identity should not be nil")
assert.Equal(t, "anonymous", anonIdentity.Name)
}
+13 -19
View File
@@ -18,7 +18,6 @@ type FilerClient interface {
type S3Authenticator interface {
AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode)
DefaultAllow() bool
}
// Server implements the Iceberg REST Catalog API.
@@ -129,25 +128,20 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
identityName, identity, errCode := s.authenticator.AuthenticateRequest(r)
if errCode != s3err.ErrNone {
// If authentication failed but DefaultAllow is enabled, proceed without identity
if s.authenticator.DefaultAllow() {
glog.V(2).Infof("Iceberg: AuthenticateRequest failed (%v), but DefaultAllow is true, proceeding", errCode)
} else {
apiErr := s3err.GetAPIError(errCode)
errorType := "RESTException"
switch apiErr.HTTPStatusCode {
case http.StatusForbidden:
errorType = "ForbiddenException"
case http.StatusUnauthorized:
errorType = "NotAuthorizedException"
case http.StatusBadRequest:
errorType = "BadRequestException"
case http.StatusInternalServerError:
errorType = "InternalServerError"
}
writeError(w, apiErr.HTTPStatusCode, errorType, apiErr.Description)
return
apiErr := s3err.GetAPIError(errCode)
errorType := "RESTException"
switch apiErr.HTTPStatusCode {
case http.StatusForbidden:
errorType = "ForbiddenException"
case http.StatusUnauthorized:
errorType = "NotAuthorizedException"
case http.StatusBadRequest:
errorType = "BadRequestException"
case http.StatusInternalServerError:
errorType = "InternalServerError"
}
writeError(w, apiErr.HTTPStatusCode, errorType, apiErr.Description)
return
}
if identityName != "" || identity != nil {
-9
View File
@@ -44,7 +44,6 @@ type IAMIntegration interface {
AuthorizeAction(ctx context.Context, identity *IAMIdentity, action Action, bucket string, objectKey string, r *http.Request) s3err.ErrorCode
ValidateSessionToken(ctx context.Context, token string) (*sts.SessionInfo, error)
ValidateTrustPolicyForPrincipal(ctx context.Context, roleArn, principalArn string) error
DefaultAllow() bool
}
// S3IAMIntegration provides IAM integration for S3 API
@@ -311,14 +310,6 @@ func (s3iam *S3IAMIntegration) ValidateTrustPolicyForPrincipal(ctx context.Conte
return s3iam.iamManager.ValidateTrustPolicyForPrincipal(ctx, roleArn, principalArn)
}
// DefaultAllow returns whether access is allowed by default when no policy is found
func (s3iam *S3IAMIntegration) DefaultAllow() bool {
if s3iam.iamManager == nil {
return true // Default to true if IAM is not enabled
}
return s3iam.iamManager.DefaultAllow()
}
// IAMIdentity represents an authenticated identity with session information
type IAMIdentity struct {
Name string
+2 -50
View File
@@ -5,7 +5,6 @@ import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
@@ -20,13 +19,9 @@ import (
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"golang.org/x/crypto/hkdf"
"google.golang.org/grpc"
)
// SSE-S3 uses AES-256 encryption with server-managed keys
@@ -457,27 +452,6 @@ func (km *SSES3KeyManager) GetKey(keyID string) (*SSES3Key, bool) {
return nil, false
}
// GetMasterKey returns a derived key from the master KEK for STS signing
// This uses HKDF to isolate the STS security domain from the SSE-S3 domain
func (km *SSES3KeyManager) GetMasterKey() []byte {
km.mu.RLock()
defer km.mu.RUnlock()
if len(km.superKey) == 0 {
return nil
}
// Derive a separate key for STS to isolate security domains
// We use the KEK as the secret, and "seaweedfs-sts-signing-key" as the info
hkdfReader := hkdf.New(sha256.New, km.superKey, nil, []byte("seaweedfs-sts-signing-key"))
derived := make([]byte, 32) // 256-bit derived key
if _, err := io.ReadFull(hkdfReader, derived); err != nil {
glog.Errorf("Failed to derive STS key: %v", err)
return nil
}
return derived
}
// Global SSE-S3 key manager instance
var globalSSES3KeyManager = NewSSES3KeyManager()
@@ -486,31 +460,9 @@ func GetSSES3KeyManager() *SSES3KeyManager {
return globalSSES3KeyManager
}
// KeyManagerFilerClient wraps wdclient.FilerClient to satisfy filer_pb.FilerClient interface
type KeyManagerFilerClient struct {
*wdclient.FilerClient
grpcDialOption grpc.DialOption
}
func (k *KeyManagerFilerClient) AdjustedUrl(location *filer_pb.Location) string {
return location.Url
}
func (k *KeyManagerFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
filerAddress := k.GetCurrentFiler()
if filerAddress == "" {
return fmt.Errorf("no filer available")
}
return pb.WithGrpcFilerClient(streamingMode, 0, filerAddress, k.grpcDialOption, fn)
}
// InitializeGlobalSSES3KeyManager initializes the global key manager with filer access
func InitializeGlobalSSES3KeyManager(filerClient *wdclient.FilerClient, grpcDialOption grpc.DialOption) error {
wrapper := &KeyManagerFilerClient{
FilerClient: filerClient,
grpcDialOption: grpcDialOption,
}
return globalSSES3KeyManager.InitializeWithFiler(wrapper)
func InitializeGlobalSSES3KeyManager(s3ApiServer *S3ApiServer) error {
return globalSSES3KeyManager.InitializeWithFiler(s3ApiServer)
}
// ProcessSSES3Request processes an SSE-S3 request and returns encryption metadata
+37 -41
View File
@@ -19,6 +19,21 @@ const (
deleteMultipleObjectsLimit = 1000
)
// objectLockVersionToCheckForDelete resolves which version should be validated for Object Lock protections.
// For versioned delete without explicit versionId, this targets the latest version (empty versionId for enabled,
// "null" for suspended) because DeleteObject affects that version's visibility.
// This always returns a version ID to check; object lock protections are fail-closed.
func objectLockVersionToCheckForDelete(versioningState, requestedVersionID string) string {
if requestedVersionID != "" {
return requestedVersionID
}
if versioningState == s3_constants.VersioningSuspended {
return "null"
}
return ""
}
func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
bucket, object := s3_constants.GetBucketAndObject(r)
@@ -52,18 +67,18 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
}
lockCheckVersionID := objectLockVersionToCheckForDelete(versioningState, versionId)
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, lockCheckVersionID, governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object, lockCheckVersionID, err)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
if versioningConfigured {
// Handle versioned delete based on specific versioning state
if versionId != "" {
// Delete specific version (same for both enabled and suspended)
// Check object lock permissions before deleting specific version
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
// Delete specific version
err := s3a.deleteSpecificObjectVersion(bucket, object, versionId)
if err != nil {
@@ -77,9 +92,7 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
} else {
// Delete without version ID - behavior depends on versioning state
if versioningEnabled {
// Enabled versioning: Create delete marker (logical delete)
// AWS S3 behavior: Delete marker creation is NOT blocked by object retention
// because it's a logical delete that doesn't actually remove the retained version
// Enabled versioning: create delete marker (logical delete)
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
if err != nil {
glog.Errorf("Failed to create delete marker: %v", err)
@@ -94,14 +107,6 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
// Suspended versioning: Actually delete the "null" version object
glog.V(2).Infof("DeleteObjectHandler: deleting null version for suspended versioning %s/%s", bucket, object)
// Check object lock permissions before deleting "null" version
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, "null", governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
// Delete the "null" version (the regular file)
err := s3a.deleteSpecificObjectVersion(bucket, object, "null")
if err != nil {
@@ -116,14 +121,6 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
}
} else {
// Handle regular delete (non-versioned)
// Check object lock permissions before deleting object
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
// Normalize trailing-slash object keys (e.g. "path/") to the
// underlying directory entry path so DeleteEntry gets a valid name.
target := util.NewFullPath(s3a.bucketDir(bucket), object)
@@ -249,20 +246,19 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
continue
}
// Check object lock permissions before deletion (only for versioned buckets)
if versioningConfigured {
// Validate governance bypass for this specific object
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key)
if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, object.VersionId, governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object.Key, object.VersionId, err)
deleteErrors = append(deleteErrors, DeleteError{
Code: s3err.GetAPIError(s3err.ErrAccessDenied).Code,
Message: s3err.GetAPIError(s3err.ErrAccessDenied).Description,
Key: object.Key,
VersionId: object.VersionId,
})
continue
}
// Check object lock permissions before deletion (applies to all buckets: versioned or non-versioned)
lockCheckVersionID := objectLockVersionToCheckForDelete(versioningState, object.VersionId)
// Validate governance bypass for this specific object
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key)
if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, lockCheckVersionID, governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object.Key, lockCheckVersionID, err)
deleteErrors = append(deleteErrors, DeleteError{
Code: s3err.GetAPIError(s3err.ErrAccessDenied).Code,
Message: s3err.GetAPIError(s3err.ErrAccessDenied).Description,
Key: object.Key,
VersionId: object.VersionId,
})
continue
}
var deleteVersionId string
@@ -0,0 +1,61 @@
package s3api
import (
"testing"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/stretchr/testify/assert"
)
func TestObjectLockVersionToCheckForDelete(t *testing.T) {
tests := []struct {
name string
versioningState string
requestedVersionID string
expectedVersionID string
}{
{
name: "enabled versioning without version id checks latest version",
versioningState: s3_constants.VersioningEnabled,
requestedVersionID: "",
expectedVersionID: "",
},
{
name: "suspended versioning without version id checks null version",
versioningState: s3_constants.VersioningSuspended,
requestedVersionID: "",
expectedVersionID: "null",
},
{
name: "specific version id is always checked",
versioningState: s3_constants.VersioningEnabled,
requestedVersionID: "3LgYQ7f7VxQ3",
expectedVersionID: "3LgYQ7f7VxQ3",
},
{
name: "non-versioned buckets still check current object",
versioningState: "",
requestedVersionID: "",
expectedVersionID: "",
},
{
name: "unknown versioning state defaults to empty version",
versioningState: "UnexpectedState",
requestedVersionID: "",
expectedVersionID: "",
},
{
name: "suspended versioning with specific version id checks that version",
versioningState: s3_constants.VersioningSuspended,
requestedVersionID: "abc123",
expectedVersionID: "abc123",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
versionID := objectLockVersionToCheckForDelete(tc.versioningState, tc.requestedVersionID)
assert.Equal(t, tc.expectedVersionID, versionID)
})
}
}
+3 -3
View File
@@ -13,7 +13,7 @@ import (
func TestGetRequestDataReader_ChunkedEncodingWithoutIAM(t *testing.T) {
// Create an S3ApiServer with IAM disabled
s3a := &S3ApiServer{
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, string(credential.StoreTypeMemory)),
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
}
// Ensure IAM is disabled for this test
s3a.iam.isAuthEnabled = false
@@ -87,7 +87,7 @@ func TestGetRequestDataReader_ChunkedEncodingWithoutIAM(t *testing.T) {
func TestGetRequestDataReader_AuthTypeDetection(t *testing.T) {
// Create an S3ApiServer with IAM disabled
s3a := &S3ApiServer{
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, string(credential.StoreTypeMemory)),
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
}
s3a.iam.isAuthEnabled = false
@@ -122,7 +122,7 @@ func TestGetRequestDataReader_AuthTypeDetection(t *testing.T) {
func TestGetRequestDataReader_IAMEnabled(t *testing.T) {
// Create an S3ApiServer with IAM enabled
s3a := &S3ApiServer{
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, string(credential.StoreTypeMemory)),
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
}
s3a.iam.isAuthEnabled = true
+40 -94
View File
@@ -110,8 +110,7 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
option.AllowedOrigins = domains
}
// Initialize basic/legacy IAM - filerClient not available yet, passed as nil
iam := NewIdentityAccessManagementWithStore(option, nil, explicitStore)
iam := NewIdentityAccessManagementWithStore(option, explicitStore)
// Initialize bucket policy engine first
policyEngine := NewBucketPolicyEngine()
@@ -147,24 +146,16 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
glog.V(1).Infof("S3 API initialized FilerClient with %d filer(s) (no discovery)", len(option.Filers))
}
// Initialize Global SSE-S3 Key Manager early so it's available for IAM fallback
// This ensures we can access the KEK for STS signing key if needed
if err := InitializeGlobalSSES3KeyManager(filerClient, option.GrpcDialOption); err != nil {
glog.Errorf("Failed to initialize SSE-S3 Key Manager: %v", err)
// We continue, as this might be a transient failure or non-critical for some setups,
// but IAM fallback to KEK will fail if this didn't succeed.
}
// Update credential store to use FilerClient's current filer for HA
iam.SetFilerClient(filerClient)
// Keep attempting to load configuration from filer now that we have a client
// The initial load in NewIdentityAccessManagementWithStore might have failed if client was nil
go func() {
if err := iam.loadS3ApiConfigurationFromFiler(option); err != nil {
glog.Warningf("Failed to load IAM config from filer after client update: %v", err)
if store := iam.credentialManager.GetStore(); store != nil {
if filerFuncSetter, ok := store.(interface {
SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption)
}); ok {
// Use FilerClient's GetCurrentFiler for true HA
filerFuncSetter.SetFilerAddressFunc(filerClient.GetCurrentFiler, option.GrpcDialOption)
glog.V(1).Infof("Updated credential store to use FilerClient's current active filer (HA-aware)")
}
}()
}
s3ApiServer = &S3ApiServer{
option: option,
@@ -187,25 +178,19 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
// This avoids circular dependency by not passing the entire S3ApiServer
iam.policyEngine = policyEngine
// Initialize advanced IAM system if config is provided or explicitly enabled
if option.IamConfig != "" || option.EnableIam {
configSource := "defaults"
if option.IamConfig != "" {
configSource = option.IamConfig
}
glog.V(1).Infof("Loading advanced IAM configuration from: %s", configSource)
// Initialize advanced IAM system if config is provided
if option.IamConfig != "" {
glog.V(1).Infof("Loading advanced IAM configuration from: %s", option.IamConfig)
// Use FilerClient's GetCurrentFiler for HA-aware filer selection
iamManager, err := loadIAMManagerFromConfig(option.IamConfig, func() string {
return string(filerClient.GetCurrentFiler())
}, func() string {
return signingKey
})
if err != nil {
glog.Errorf("Failed to load IAM configuration: %v", err)
} else {
if s3ApiServer.iam.credentialManager != nil {
iamManager.SetUserStore(s3ApiServer.iam.credentialManager)
if iam.credentialManager != nil {
iamManager.SetUserStore(iam.credentialManager)
}
glog.V(1).Infof("IAM Manager loaded, creating integration")
// Create S3 IAM integration with the loaded IAM manager
@@ -248,10 +233,6 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
})
}
s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer)
// Update IAM with the final filer client (already handled by SetFilerClient above,
// but this reinforces it if we ever change the flow)
s3ApiServer.iam.SetFilerClient(s3ApiServer.filerClient)
if option.LocalFilerSocket == "" {
if s3ApiServer.client, err = util_http.NewGlobalHttpClient(); err != nil {
return nil, err
@@ -268,6 +249,11 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
s3ApiServer.registerRouter(router)
// Initialize the global SSE-S3 key manager with filer access
if err := InitializeGlobalSSES3KeyManager(s3ApiServer); err != nil {
return nil, fmt.Errorf("failed to initialize SSE-S3 key manager: %w", err)
}
go s3ApiServer.subscribeMetaEvents("s3", startTsNs, filer.DirectoryEtcRoot, []string{
option.BucketsPath,
filer.IamConfigDirectory,
@@ -844,7 +830,14 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
}
// loadIAMManagerFromConfig loads the advanced IAM manager from configuration file
func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() string, getFilerSigningKey func() string) (*integration.IAMManager, error) {
func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() string) (*integration.IAMManager, error) {
// Read configuration file
configData, err := os.ReadFile(configPath)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
// Parse configuration structure
var configRoot struct {
STS *sts.STSConfig `json:"sts"`
Policy *policy.PolicyEngineConfig `json:"policy"`
@@ -856,43 +849,24 @@ func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() str
} `json:"policies"`
}
if configPath != "" {
// Read configuration file
configData, err := os.ReadFile(configPath)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
if err := json.Unmarshal(configData, &configRoot); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
} else {
glog.V(1).Infof("No IAM config file provided; using defaults")
// Initialize with empty config which will trigger defaults below
}
// Ensure STS config exists so we can apply defaults later
if configRoot.STS == nil {
configRoot.STS = &sts.STSConfig{}
if err := json.Unmarshal(configData, &configRoot); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
// Ensure a valid policy engine config exists
if configRoot.Policy == nil {
configRoot.Policy = &policy.PolicyEngineConfig{}
}
if configRoot.Policy.StoreType == "" {
configRoot.Policy.StoreType = sts.StoreTypeMemory
}
if configRoot.Policy.DefaultEffect == "" {
// Default to Allow (open) with in-memory store so that
// users can start using STS without locking themselves out immediately.
// For other stores (e.g. filer), default to Deny (closed) for security.
if configRoot.Policy.StoreType == sts.StoreTypeMemory {
configRoot.Policy.DefaultEffect = sts.EffectAllow
} else {
configRoot.Policy.DefaultEffect = sts.EffectDeny
// Provide a secure default if not specified in the config file
// Default to Deny with in-memory store so that JSON-defined policies work without filer
glog.V(1).Infof("No policy engine config provided; using defaults (DefaultEffect=%s, StoreType=%s)", sts.EffectDeny, sts.StoreTypeMemory)
configRoot.Policy = &policy.PolicyEngineConfig{
DefaultEffect: sts.EffectDeny,
StoreType: sts.StoreTypeMemory,
}
glog.V(1).Infof("Using policy defaults: DefaultEffect=%s, StoreType=%s", configRoot.Policy.DefaultEffect, configRoot.Policy.StoreType)
} else if configRoot.Policy.StoreType == "" {
// If policy config exists but storeType is not specified, use memory store
// This ensures JSON-defined policies are stored in memory and work correctly
configRoot.Policy.StoreType = sts.StoreTypeMemory
glog.V(1).Infof("Policy storeType not specified; using memory store for JSON config-based setup")
}
// Create IAM configuration
@@ -904,26 +878,6 @@ func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() str
},
}
// Apply default signing key if not present in config
if iamConfig.STS != nil && len(iamConfig.STS.SigningKey) == 0 {
// 1. Try server-configured signing key (security.toml / CLI)
if key := getFilerSigningKey(); key != "" {
iamConfig.STS.SigningKey = []byte(key)
glog.V(1).Infof("Using default filer signing key for STS service")
} else {
// 2. Try cluster-wide SSE-S3 Master Key (KEK) from Filer
// This ensures zero-config consistency across the cluster
if kek := GetSSES3KeyManager().GetMasterKey(); len(kek) > 0 {
iamConfig.STS.SigningKey = kek
glog.V(1).Infof("Using SSE-S3 Master Key (KEK) for STS service")
} else {
// 3. Fail if no signing key is available
// This ensures consistency across multiple S3 servers and secure operation
return nil, fmt.Errorf("no signing key found for STS service; please provide 'signingKey' in IAM config, configure 'jwt.filer_signing.key' in security.toml, or ensure SSE-S3 is initialized")
}
}
}
// Initialize IAM manager
iamManager := integration.NewIAMManager()
if err := iamManager.Initialize(iamConfig, filerAddressProvider); err != nil {
@@ -1006,11 +960,3 @@ func (s3a *S3ApiServer) AuthenticateRequest(r *http.Request) (string, interface{
}
return "", nil, err
}
// DefaultAllow returns whether access is allowed by default when no policy is found
func (s3a *S3ApiServer) DefaultAllow() bool {
if s3a.iam == nil || s3a.iam.iamIntegration == nil {
return false
}
return s3a.iam.iamIntegration.DefaultAllow()
}
+1 -1
View File
@@ -16,7 +16,7 @@ import (
// setupRoutingTestServer creates a minimal S3ApiServer for routing tests
func setupRoutingTestServer(t *testing.T) *S3ApiServer {
opt := &S3ApiServerOption{EnableIam: true}
iam := NewIdentityAccessManagementWithStore(opt, nil, "memory")
iam := NewIdentityAccessManagementWithStore(opt, "memory")
iam.isAuthEnabled = true
if iam.credentialManager == nil {
+3 -20
View File
@@ -43,11 +43,6 @@ func (st *S3TablesApiServer) SetAccountID(accountID string) {
st.handler.SetAccountID(accountID)
}
// SetDefaultAllow sets whether to allow access by default
func (st *S3TablesApiServer) SetDefaultAllow(allow bool) {
st.handler.SetDefaultAllow(allow)
}
// S3TablesHandler handles S3 Tables API requests
func (st *S3TablesApiServer) S3TablesHandler(w http.ResponseWriter, r *http.Request) {
st.handler.HandleRequest(w, r, st)
@@ -62,12 +57,6 @@ func (st *S3TablesApiServer) WithFilerClient(streamingMode bool, fn func(filer_p
func (s3a *S3ApiServer) registerS3TablesRoutes(router *mux.Router) {
// Create S3 Tables handler
s3TablesApi := NewS3TablesApiServer(s3a)
if s3a.iam != nil && s3a.iam.iamIntegration != nil {
s3TablesApi.SetDefaultAllow(s3a.iam.iamIntegration.DefaultAllow())
} else {
// If IAM is not configured, allow all access by default
s3TablesApi.SetDefaultAllow(true)
}
// Regex for S3 Tables Bucket ARN
const tableBucketARNRegex = "arn:aws:s3tables:[^/:]*:[^/:]*:bucket/[^/]+"
@@ -629,15 +618,9 @@ func (s3a *S3ApiServer) authenticateS3Tables(f http.HandlerFunc) http.HandlerFun
// Use AuthSignatureOnly to authenticate the request without authorizing specific actions
identity, errCode := s3a.iam.AuthSignatureOnly(r)
if errCode != s3err.ErrNone {
// If IAM is enabled but DefaultAllow is true, we can proceed even if unauthenticated
// authorization checks in handlers will then use DefaultAllow logic.
if s3a.iam.iamIntegration != nil && s3a.iam.iamIntegration.DefaultAllow() {
glog.V(2).Infof("S3Tables: AuthSignatureOnly failed (%v), but DefaultAllow is true, proceeding", errCode)
} else {
glog.Errorf("S3Tables: AuthSignatureOnly failed: %v", errCode)
s3err.WriteErrorResponse(w, r, errCode)
return
}
glog.Errorf("S3Tables: AuthSignatureOnly failed: %v", errCode)
s3err.WriteErrorResponse(w, r, errCode)
return
}
// Store the authenticated identity in request context
+4 -11
View File
@@ -44,17 +44,15 @@ const (
// S3TablesHandler handles S3 Tables API requests
type S3TablesHandler struct {
region string
accountID string
defaultAllow bool // Whether to allow access by default (for zero-config IAM)
region string
accountID string
}
// NewS3TablesHandler creates a new S3 Tables handler
func NewS3TablesHandler() *S3TablesHandler {
return &S3TablesHandler{
region: DefaultRegion,
accountID: DefaultAccountID,
defaultAllow: false,
region: DefaultRegion,
accountID: DefaultAccountID,
}
}
@@ -72,11 +70,6 @@ func (h *S3TablesHandler) SetAccountID(accountID string) {
}
}
// SetDefaultAllow sets whether to allow access by default
func (h *S3TablesHandler) SetDefaultAllow(allow bool) {
h.defaultAllow = allow
}
// FilerClient interface for filer operations
type FilerClient interface {
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
+1 -3
View File
@@ -16,9 +16,7 @@ import (
func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
// Check permission
principal := h.getAccountID(r)
if !CheckPermissionWithContext("CreateTableBucket", principal, principal, "", "", &PolicyContext{
DefaultAllow: h.defaultAllow,
}) {
if !CanCreateTableBucket(principal, principal, "") {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
}
@@ -72,7 +72,6 @@ func (h *S3TablesHandler) handleGetTableBucket(w http.ResponseWriter, r *http.Re
if !CheckPermissionWithContext("GetTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket details")
return ErrAccessDenied
@@ -102,7 +101,6 @@ func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.
identityActions := getIdentityActions(r)
if !CheckPermissionWithContext("ListTableBuckets", principal, accountID, "", "", &PolicyContext{
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to list table buckets")
return NewAuthError("ListTableBuckets", principal, "not authorized to list table buckets")
@@ -200,7 +198,6 @@ func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.
if !CheckPermissionWithContext("GetTableBucket", accountID, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: entry.Entry.Name,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
continue
}
@@ -303,7 +300,6 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http
if !CheckPermissionWithContext("DeleteTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
return NewAuthError("DeleteTableBucket", principal, fmt.Sprintf("not authorized to delete bucket %s", bucketName))
}
-4
View File
@@ -118,7 +118,6 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
glog.Infof("S3Tables: Permission denied for CreateNamespace - principal=%s, owner=%s", principal, bucketMetadata.OwnerAccountID)
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create namespace in this bucket")
@@ -259,7 +258,6 @@ func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Requ
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
return ErrAccessDenied
@@ -346,7 +344,6 @@ func (h *S3TablesHandler) handleListNamespaces(w http.ResponseWriter, r *http.Re
TableBucketName: bucketName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
return ErrAccessDenied
@@ -531,7 +528,6 @@ func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.R
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
return ErrAccessDenied
-9
View File
@@ -94,7 +94,6 @@ func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *h
if !CheckPermissionWithContext("PutTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, "", bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table bucket policy")
return NewAuthError("PutTableBucketPolicy", principal, "not authorized to put table bucket policy")
@@ -172,7 +171,6 @@ func (h *S3TablesHandler) handleGetTableBucketPolicy(w http.ResponseWriter, r *h
if !CheckPermissionWithContext("GetTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, string(policy), bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket policy")
return NewAuthError("GetTableBucketPolicy", principal, "not authorized to get table bucket policy")
@@ -248,7 +246,6 @@ func (h *S3TablesHandler) handleDeleteTableBucketPolicy(w http.ResponseWriter, r
if !CheckPermissionWithContext("DeleteTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table bucket policy")
return NewAuthError("DeleteTableBucketPolicy", principal, "not authorized to delete table bucket policy")
@@ -349,7 +346,6 @@ func (h *S3TablesHandler) handlePutTablePolicy(w http.ResponseWriter, r *http.Re
Namespace: namespaceName,
TableName: tableName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table policy")
return NewAuthError("PutTablePolicy", principal, "not authorized to put table policy")
@@ -457,7 +453,6 @@ func (h *S3TablesHandler) handleGetTablePolicy(w http.ResponseWriter, r *http.Re
Namespace: namespaceName,
TableName: tableName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table policy")
return NewAuthError("GetTablePolicy", principal, "not authorized to get table policy")
@@ -547,7 +542,6 @@ func (h *S3TablesHandler) handleDeleteTablePolicy(w http.ResponseWriter, r *http
Namespace: namespaceName,
TableName: tableName,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table policy")
return NewAuthError("DeleteTablePolicy", principal, "not authorized to delete table policy")
@@ -646,7 +640,6 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque
TagKeys: requestTagKeys,
ResourceTags: existingTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
return NewAuthError("TagResource", principal, "not authorized to tag resource")
}
@@ -764,7 +757,6 @@ func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *ht
TableBucketTags: bucketTags,
ResourceTags: tags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
return NewAuthError("ListTagsForResource", principal, "not authorized to list tags for resource")
}
@@ -872,7 +864,6 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req
TagKeys: req.TagKeys,
ResourceTags: tags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
return NewAuthError("UntagResource", principal, "not authorized to untag resource")
}
-11
View File
@@ -145,7 +145,6 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
TagKeys: mapKeys(req.Tags),
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
bucketAllowed := CheckPermissionWithContext("CreateTable", accountID, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
@@ -155,7 +154,6 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
TagKeys: mapKeys(req.Tags),
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
if !nsAllowed && !bucketAllowed {
@@ -392,7 +390,6 @@ func (h *S3TablesHandler) handleGetTable(w http.ResponseWriter, r *http.Request,
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
bucketAllowed := CheckPermissionWithContext("GetTable", accountID, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
@@ -401,7 +398,6 @@ func (h *S3TablesHandler) handleGetTable(w http.ResponseWriter, r *http.Request,
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
if !tableAllowed && !bucketAllowed {
@@ -531,14 +527,12 @@ func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Reques
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
bucketAllowed := CheckPermissionWithContext("ListTables", accountID, bucketMeta.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
Namespace: namespaceName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
if !nsAllowed && !bucketAllowed {
return ErrAccessDenied
@@ -580,7 +574,6 @@ func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Reques
TableBucketName: bucketName,
TableBucketTags: bucketTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
}) {
return ErrAccessDenied
}
@@ -917,7 +910,6 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
bucketAllowed := CheckPermissionWithContext("DeleteTable", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
@@ -926,7 +918,6 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
if !tableAllowed && !bucketAllowed {
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table")
@@ -1062,7 +1053,6 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
bucketAllowed := CheckPermissionWithContext("UpdateTable", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
TableBucketName: bucketName,
@@ -1071,7 +1061,6 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
TableBucketTags: bucketTags,
ResourceTags: tableTags,
IdentityActions: identityActions,
DefaultAllow: h.defaultAllow,
})
if !tableAllowed && !bucketAllowed {
+1 -9
View File
@@ -20,10 +20,7 @@ type Manager struct {
// NewManager creates a new Manager.
func NewManager() *Manager {
m := &Manager{handler: NewS3TablesHandler()}
// Default to allowing access when IAM is not configured
m.handler.SetDefaultAllow(true)
return m
return &Manager{handler: NewS3TablesHandler()}
}
// SetRegion sets the AWS region for ARN generation.
@@ -36,11 +33,6 @@ func (m *Manager) SetAccountID(accountID string) {
m.handler.SetAccountID(accountID)
}
// SetDefaultAllow sets whether to allow access by default.
func (m *Manager) SetDefaultAllow(allow bool) {
m.handler.SetDefaultAllow(allow)
}
// Execute runs an S3 Tables operation and decodes the response into resp (if provided).
func (m *Manager) Execute(ctx context.Context, filerClient FilerClient, operation string, req interface{}, resp interface{}, identity string) error {
body, err := json.Marshal(req)
+4 -19
View File
@@ -86,7 +86,6 @@ type PolicyContext struct {
SSEAlgorithm string
KMSKeyArn string
StorageClass string
DefaultAllow bool
}
// CheckPermissionWithResource checks if a principal has permission to perform an operation on a specific resource
@@ -118,7 +117,6 @@ func CheckPermissionWithContext(operation, principal, owner, resourcePolicy, res
}
func checkPermission(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
// Owner always has permission
if principal == owner {
return true
@@ -128,11 +126,8 @@ func checkPermission(operation, principal, owner, resourcePolicy, resourceARN st
return true
}
// If no policy is provided, use default allow if enabled
// If no policy is provided, deny access (default deny)
if resourcePolicy == "" {
if ctx != nil && ctx.DefaultAllow {
return true
}
return false
}
@@ -174,25 +169,15 @@ func checkPermission(operation, principal, owner, resourcePolicy, resourceARN st
}
// Statement matches - check effect
switch stmt.Effect {
case "Allow":
if stmt.Effect == "Allow" {
hasAllow = true
case "Deny":
} else if stmt.Effect == "Deny" {
// Explicit deny always wins
return false
}
}
if hasAllow {
return true
}
// If no statement matched, use default allow if enabled
if ctx != nil && ctx.DefaultAllow {
return true
}
return false
return hasAllow
}
func hasIdentityPermission(operation string, ctx *PolicyContext) bool {
-45
View File
@@ -206,48 +206,3 @@ func TestEvaluatePolicyWithConditions(t *testing.T) {
})
}
}
func TestCheckPermissionWithDefaultAllow(t *testing.T) {
tests := []struct {
name string
defaultAllow bool
policy string
expected bool
}{
{
"default deny (no policy, DefaultAllow=false)",
false,
"",
false,
},
{
"default allow (no policy, DefaultAllow=true)",
true,
"",
true,
},
{
"explicit deny overrides DefaultAllow=true",
true,
`{"Statement":[{"Effect":"Deny","Principal":"*","Action":"s3tables:GetTable"}]}`,
false,
},
{
"explicit allow works with DefaultAllow=false",
false,
`{"Statement":[{"Effect":"Allow","Principal":"*","Action":"s3tables:GetTable"}]}`,
true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := &PolicyContext{
DefaultAllow: tt.defaultAllow,
}
result := CheckPermissionWithContext("s3tables:GetTable", "user123", "owner123", tt.policy, "", ctx)
if result != tt.expected {
t.Errorf("CheckPermissionWithContext() = %v, want %v (DefaultAllow=%v, Policy=%s)", result, tt.expected, tt.defaultAllow, tt.policy)
}
})
}
}
-3
View File
@@ -41,9 +41,6 @@ func (m *mockIAMIntegration) ValidateTrustPolicyForPrincipal(ctx context.Context
func (m *mockIAMIntegration) ValidateSessionToken(ctx context.Context, token string) (*sts.SessionInfo, error) {
return nil, nil
}
func (m *mockIAMIntegration) DefaultAllow() bool {
return true
}
func TestSTSAssumeRolePostBody(t *testing.T) {
// Setup S3ApiServer with IAM enabled
-2
View File
@@ -13,7 +13,6 @@ import (
"sync"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/telemetry"
@@ -313,7 +312,6 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
func (ms *MasterServer) startAdminScripts() {
v := util.GetViper()
v.SetDefault("master.maintenance.scripts", maintenance.DefaultMasterMaintenanceScripts)
adminScripts := v.GetString("master.maintenance.scripts")
if adminScripts == "" {
return
+6 -1
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
)
@@ -94,7 +95,7 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
case volume_server_pb.VolumeScrubMode_LOCAL:
files, shardInfos, serrs = v.ScrubLocal()
case volume_server_pb.VolumeScrubMode_FULL:
files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId)
files, shardInfos, serrs = scrubEcVolumeFull(ctx, v)
default:
return nil, fmt.Errorf("unsupported EC volume scrub mode %d", m)
}
@@ -119,3 +120,7 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
}
return res, nil
}
func scrubEcVolumeFull(ctx context.Context, v *erasure_coding.EcVolume) (int64, []*volume_server_pb.EcShardInfo, []error) {
return 0, nil, []error{fmt.Errorf("scrubEcVolumeFull(): not implemented, see https://github.com/seaweedfs/seaweedfs/issues/8018")}
}
-17
View File
@@ -43,23 +43,6 @@ func AllShardIds() []ShardId {
return res
}
// Compares a pair of EcShardInfo protos for sorting.
func CmpEcShardInfo(a, b *volume_server_pb.EcShardInfo) int {
if a.VolumeId < b.VolumeId {
return -1
}
if a.VolumeId > b.VolumeId {
return 1
}
if a.ShardId < b.ShardId {
return -1
}
if a.ShardId > b.ShardId {
return 1
}
return 0
}
type EcVolumeShard struct {
VolumeId needle.VolumeId
ShardId ShardId
+26 -45
View File
@@ -159,7 +159,7 @@ func (s *Store) ReadEcShardNeedle(vid needle.VolumeId, n *needle.Needle, onReadS
if len(intervals) > 1 {
glog.V(3).Infof("ReadEcShardNeedle needle id %s intervals:%+v", n.String(), intervals)
}
bytes, isDeleted, err := s.readEcShardIntervals(n.Id, localEcVolume, intervals)
bytes, isDeleted, err := s.readEcShardIntervals(vid, n.Id, localEcVolume, intervals)
if err != nil {
return 0, fmt.Errorf("ReadEcShardIntervals: %w", err)
}
@@ -178,11 +178,8 @@ func (s *Store) ReadEcShardNeedle(vid needle.VolumeId, n *needle.Needle, onReadS
return 0, fmt.Errorf("ec shard %d not found", vid)
}
func (s *Store) IntervalToShardIdAndOffset(iv erasure_coding.Interval) (erasure_coding.ShardId, int64) {
return iv.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
}
func (s *Store) readEcShardIntervals(vid needle.VolumeId, needleId types.NeedleId, ecVolume *erasure_coding.EcVolume, intervals []erasure_coding.Interval) (data []byte, is_deleted bool, err error) {
func (s *Store) readEcShardIntervals(needleId types.NeedleId, ecVolume *erasure_coding.EcVolume, intervals []erasure_coding.Interval) (data []byte, is_deleted bool, err error) {
if err = s.cachedLookupEcShardLocations(ecVolume); err != nil {
return nil, false, fmt.Errorf("failed to locate shard via master grpc %s: %v", s.MasterAddress, err)
}
@@ -205,36 +202,37 @@ func (s *Store) readEcShardIntervals(needleId types.NeedleId, ecVolume *erasure_
}
func (s *Store) readOneEcShardInterval(needleId types.NeedleId, ecVolume *erasure_coding.EcVolume, interval erasure_coding.Interval) (data []byte, is_deleted bool, err error) {
shardId, actualOffset := s.IntervalToShardIdAndOffset(interval)
shardId, actualOffset := interval.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
data = make([]byte, interval.Size)
if shard, found := ecVolume.FindEcVolumeShard(shardId); found {
var readSize int
if readSize, err = shard.ReadAt(data, actualOffset); err != nil {
if readSize != int(interval.Size) {
glog.V(0).Infof("read local ec shard %d.%d offset %d: %v", ecVolume.VolumeId, shardId, actualOffset, err)
return
}
}
} else {
ecVolume.ShardLocationsLock.RLock()
sourceDataNodes, hasShardIdLocation := ecVolume.ShardLocations[shardId]
ecVolume.ShardLocationsLock.RUnlock()
// try local read
err = s.readLocalEcShardInterval(ecVolume, shardId, data, actualOffset)
if err == nil {
return
}
glog.V(0).Infof("read local ec shard %d.%d offset %d: %v", ecVolume.VolumeId, shardId, actualOffset, err)
// try reading directly
if hasShardIdLocation {
_, is_deleted, err = s.readRemoteEcShardInterval(sourceDataNodes, needleId, ecVolume.VolumeId, shardId, data, actualOffset)
if err == nil {
return
}
glog.V(0).Infof("clearing ec shard %d.%d locations: %v", ecVolume.VolumeId, shardId, err)
}
ecVolume.ShardLocationsLock.RLock()
sourceDataNodes, hasShardIdLocation := ecVolume.ShardLocations[shardId]
ecVolume.ShardLocationsLock.RUnlock()
// try reading directly
if hasShardIdLocation {
_, is_deleted, err = s.readRemoteEcShardInterval(sourceDataNodes, needleId, ecVolume.VolumeId, shardId, data, actualOffset)
// try reading by recovering from other shards
_, is_deleted, err = s.recoverOneRemoteEcShardInterval(needleId, ecVolume, shardId, data, actualOffset)
if err == nil {
return
}
glog.V(0).Infof("read remote ec shard %d.%d locations: %v", ecVolume.VolumeId, shardId, err)
glog.V(0).Infof("recover ec shard %d.%d : %v", ecVolume.VolumeId, shardId, err)
}
// try reading by recovering from other shards
_, is_deleted, err = s.recoverOneRemoteEcShardInterval(needleId, ecVolume, shardId, data, actualOffset)
if err == nil {
return
}
glog.V(0).Infof("recover ec shard %d.%d : %v", ecVolume.VolumeId, shardId, err)
return
}
@@ -288,23 +286,6 @@ func (s *Store) cachedLookupEcShardLocations(ecVolume *erasure_coding.EcVolume)
return
}
func (s *Store) readLocalEcShardInterval(ecVolume *erasure_coding.EcVolume, shardId erasure_coding.ShardId, buf []byte, offset int64) error {
shard, found := ecVolume.FindEcVolumeShard(shardId)
if !found {
return fmt.Errorf("shard %d not found for volume %d", shardId, ecVolume.VolumeId)
}
readBytes, err := shard.ReadAt(buf, offset)
if err != nil {
return fmt.Errorf("failed to read local EC shard %d for volume %d: %v", shardId, ecVolume.VolumeId, err)
}
if got, want := readBytes, len(buf); got != want {
return fmt.Errorf("expected %d bytes for local EC shard %d on volume %d, got %d", want, shardId, ecVolume.VolumeId, got)
}
return nil
}
func (s *Store) readRemoteEcShardInterval(sourceDataNodes []pb.ServerAddress, needleId types.NeedleId, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, is_deleted bool, err error) {
if len(sourceDataNodes) == 0 {
-97
View File
@@ -1,97 +0,0 @@
package storage
import (
"fmt"
"slices"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
// ScrubEcVolume checks the full integrity of a EC volume, across both local and remote shards.
// Returns a count of processed file entries, slice of found broken shards, and slice of found errors.
func (s *Store) ScrubEcVolume(vid needle.VolumeId) (int64, []*volume_server_pb.EcShardInfo, []error) {
ecv, found := s.FindEcVolume(vid)
if !found {
return 0, nil, []error{fmt.Errorf("EC volume id %d not found", vid)}
}
if err := s.cachedLookupEcShardLocations(ecv); err != nil {
return 0, nil, []error{fmt.Errorf("failed to locate shard via master grpc %s: %v", s.MasterAddress, err)}
}
// full scan means verifying indexes as well
_, errs := ecv.ScrubIndex()
var count int64
// reads for EC chunks can hit the same shard multiple times, so dedupe upon read errors
brokenShardsMap := map[erasure_coding.ShardId]*volume_server_pb.EcShardInfo{}
err := ecv.WalkIndex(func(id types.NeedleId, offset types.Offset, size types.Size) error {
count++
if size.IsTombstone() {
// nothing to do for tombstones...
return nil
}
data := make([]byte, 0, needle.GetActualSize(size, ecv.Version))
intervals := ecv.LocateEcShardNeedleInterval(ecv.Version, offset.ToActualOffset(), size)
for i, iv := range intervals {
chunk := make([]byte, iv.Size)
shardId, offset := s.IntervalToShardIdAndOffset(iv)
// try a local shard read first...
if err := s.readLocalEcShardInterval(ecv, shardId, chunk, offset); err == nil {
data = append(data, chunk...)
continue
}
// ...then remote. note we do not try to recover EC-encoded data upon read failures;
// we want check that shards are valid without decoding
ecv.ShardLocationsLock.RLock()
sourceDataNodes, ok := ecv.ShardLocations[shardId]
ecv.ShardLocationsLock.RUnlock()
if ok {
if _, _, err := s.readRemoteEcShardInterval(sourceDataNodes, id, ecv.VolumeId, shardId, chunk, offset); err == nil {
data = append(data, chunk...)
continue
}
}
// chunk read for shard failed :(
errs = append(errs, fmt.Errorf("failed to read EC shard %d for needle %d on volume %d (interval %d/%d)", shardId, id, ecv.VolumeId, i+1, len(intervals)))
brokenShardsMap[shardId] = &volume_server_pb.EcShardInfo{
ShardId: uint32(shardId),
Size: int64(iv.Size),
Collection: ecv.Collection,
VolumeId: uint32(ecv.VolumeId),
}
break
}
if got, want := int64(len(data)), needle.GetActualSize(size, ecv.Version); got != want {
errs = append(errs, fmt.Errorf("expected %d bytes for needle %d, got %d", want, id, got))
return nil
}
n := needle.Needle{}
if err := n.ReadBytes(data, 0, size, ecv.Version); err != nil {
errs = append(errs, fmt.Errorf("needle %d on EC volume %d: %v", id, ecv.VolumeId, err))
}
return nil
})
if err != nil {
errs = append(errs, err)
}
brokenShards := []*volume_server_pb.EcShardInfo{}
for _, s := range brokenShardsMap {
brokenShards = append(brokenShards, s)
}
slices.SortFunc(brokenShards, erasure_coding.CmpEcShardInfo)
return count, brokenShards, errs
}
+1 -8
View File
@@ -16,7 +16,6 @@ func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) {
}
return 0, fmt.Errorf("volume id %d is not found during check compact", volumeId)
}
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
if v := s.findVolume(vid); v != nil {
// Get current volume size for space calculation
@@ -40,15 +39,10 @@ func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compaction
glog.V(1).Infof("volume %d compaction space check: volume=%d, index=%d, space_needed=%d, free_space=%d",
vid, volumeSize, indexSize, spaceNeeded, diskStatus.Free)
return v.CompactByIndex(&CompactOptions{
PreallocateBytes: preallocate,
MaxBytesPerSecond: compactionBytePerSecond,
ProgressCallback: progressFn,
})
return v.Compact2(preallocate, compactionBytePerSecond, progressFn)
}
return fmt.Errorf("volume id %d is not found during compact", vid)
}
func (s *Store) CommitCompactVolume(vid needle.VolumeId) (bool, int64, error) {
if s.isStopping {
return false, 0, fmt.Errorf("volume id %d skips compact because volume is stopping", vid)
@@ -64,7 +58,6 @@ func (s *Store) CommitCompactVolume(vid needle.VolumeId) (bool, int64, error) {
}
return false, 0, fmt.Errorf("volume id %d is not found during commit compact", vid)
}
func (s *Store) CommitCleanupVolume(vid needle.VolumeId) error {
if v := s.findVolume(vid); v != nil {
return v.cleanupCompact()
+33 -61
View File
@@ -35,26 +35,8 @@ func (v *Volume) garbageLevel() float64 {
return float64(deletedSize) / float64(fileSize)
}
type CompactOptions struct {
PreallocateBytes int64
MaxBytesPerSecond int64
ProgressCallback ProgressFunc
// internal state settings
srcDatPath string
srcIdxPath string
destDatPath string
destIdxPath string
superBlock super_block.SuperBlock
version needle.Version
}
// compact a volume based on deletions in .dat files
func (v *Volume) CompactByVolumeData(opts *CompactOptions) error {
if opts == nil {
// default settings
opts = &CompactOptions{}
}
func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error {
if v.MemoryMapMaxSizeMb != 0 { //it makes no sense to compact in memory
return nil
@@ -82,20 +64,11 @@ func (v *Volume) CompactByVolumeData(opts *CompactOptions) error {
if err := v.nm.Sync(); err != nil {
glog.V(0).Infof("compact failed to sync volume idx %d", v.Id)
}
opts.destDatPath = v.FileName(".cpd")
opts.destIdxPath = v.FileName(".cpx")
opts.superBlock = v.SuperBlock
opts.version = v.Version()
return v.copyDataAndGenerateIndexFile(opts)
return v.copyDataAndGenerateIndexFile(v.FileName(".cpd"), v.FileName(".cpx"), preallocate, compactionBytePerSecond)
}
// compact a volume based on deletions in .idx files
func (v *Volume) CompactByIndex(opts *CompactOptions) error {
if opts == nil {
// default settings
opts = &CompactOptions{}
}
func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
if v.MemoryMapMaxSizeMb != 0 { //it makes no sense to compact in memory
return nil
@@ -123,14 +96,15 @@ func (v *Volume) CompactByIndex(opts *CompactOptions) error {
if err := v.nm.Sync(); err != nil {
glog.V(0).Infof("compact2 failed to sync volume idx %d: %v", v.Id, err)
}
opts.srcDatPath = v.FileName(".dat")
opts.srcIdxPath = v.FileName(".idx")
opts.destDatPath = v.FileName(".cpd")
opts.destIdxPath = v.FileName(".cpx")
opts.superBlock = v.SuperBlock
opts.version = v.Version()
return v.copyDataBasedOnIndexFile(opts)
return v.copyDataBasedOnIndexFile(
v.FileName(".dat"), v.FileName(".idx"),
v.FileName(".cpd"), v.FileName(".cpx"),
v.SuperBlock,
v.Version(),
preallocate,
compactionBytePerSecond,
progressFn,
)
}
func (v *Volume) CommitCompact() error {
@@ -429,9 +403,9 @@ func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *needle.Needle, offset in
return nil
}
func (v *Volume) copyDataAndGenerateIndexFile(opts *CompactOptions) (err error) {
func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, preallocate int64, compactionBytePerSecond int64) (err error) {
var dst backend.BackendStorageFile
if dst, err = backend.CreateVolumeFile(opts.destDatPath, opts.PreallocateBytes, 0); err != nil {
if dst, err = backend.CreateVolumeFile(dstName, preallocate, 0); err != nil {
return err
}
defer dst.Close()
@@ -444,7 +418,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(opts *CompactOptions) (err error)
now: uint64(time.Now().Unix()),
nm: nm,
dstBackend: dst,
writeThrottler: util.NewWriteThrottler(opts.MaxBytesPerSecond),
writeThrottler: util.NewWriteThrottler(compactionBytePerSecond),
}
err = ScanVolumeFile(v.dir, v.Collection, v.Id, v.needleMapKind, scanner)
if err != nil {
@@ -452,15 +426,15 @@ func (v *Volume) copyDataAndGenerateIndexFile(opts *CompactOptions) (err error)
return err
}
return nm.SaveToIdx(opts.destIdxPath)
return nm.SaveToIdx(idxName)
}
func (v *Volume) copyDataBasedOnIndexFile(opts *CompactOptions) (err error) {
func (v *Volume) copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName string, sb super_block.SuperBlock, version needle.Version, preallocate, compactionBytePerSecond int64, progressFn ProgressFunc) (err error) {
var (
srcDatBackend, dstDatBackend backend.BackendStorageFile
dataFile *os.File
)
if dstDatBackend, err = backend.CreateVolumeFile(opts.destDatPath, opts.PreallocateBytes, 0); err != nil {
if dstDatBackend, err = backend.CreateVolumeFile(dstDatName, preallocate, 0); err != nil {
return err
}
defer func() {
@@ -472,10 +446,10 @@ func (v *Volume) copyDataBasedOnIndexFile(opts *CompactOptions) (err error) {
defer oldNm.Close()
newNm := needle_map.NewMemDb()
defer newNm.Close()
if err = oldNm.LoadFromIdx(opts.srcIdxPath); err != nil {
if err = oldNm.LoadFromIdx(srcIdxName); err != nil {
return err
}
if dataFile, err = os.Open(opts.srcDatPath); err != nil {
if dataFile, err = os.Open(srcDatName); err != nil {
return err
}
srcDatBackend = backend.NewDiskFile(dataFile)
@@ -483,13 +457,11 @@ func (v *Volume) copyDataBasedOnIndexFile(opts *CompactOptions) (err error) {
now := uint64(time.Now().Unix())
opts.superBlock.CompactionRevision++
newOffset := int64(opts.superBlock.BlockSize())
if _, err := dstDatBackend.WriteAt(opts.superBlock.Bytes(), 0); err != nil {
return fmt.Errorf("failed to write superblock: %v", err)
}
sb.CompactionRevision++
dstDatBackend.WriteAt(sb.Bytes(), 0)
newOffset := int64(sb.BlockSize())
writeThrottler := util.NewWriteThrottler(opts.MaxBytesPerSecond)
writeThrottler := util.NewWriteThrottler(compactionBytePerSecond)
err = oldNm.AscendingVisit(func(value needle_map.NeedleValue) error {
offset, size := value.Offset, value.Size
@@ -498,29 +470,29 @@ func (v *Volume) copyDataBasedOnIndexFile(opts *CompactOptions) (err error) {
return nil
}
if opts.ProgressCallback != nil {
if !opts.ProgressCallback(offset.ToActualOffset()) {
if progressFn != nil {
if !progressFn(offset.ToActualOffset()) {
return fmt.Errorf("interrupted")
}
}
n := new(needle.Needle)
if err := n.ReadData(srcDatBackend, offset.ToActualOffset(), size, opts.version); err != nil {
if err := n.ReadData(srcDatBackend, offset.ToActualOffset(), size, version); err != nil {
v.checkReadWriteError(err)
return fmt.Errorf("cannot hydrate needle from file: %w", err)
}
if n.HasTtl() && now >= n.LastModified+uint64(opts.superBlock.Ttl.Minutes()*60) {
if n.HasTtl() && now >= n.LastModified+uint64(sb.Ttl.Minutes()*60) {
return nil
}
if err = newNm.Set(n.Id, ToOffset(newOffset), n.Size); err != nil {
return fmt.Errorf("cannot put needle: %s", err)
}
if _, _, _, err = n.Append(dstDatBackend, opts.superBlock.Version); err != nil {
if _, _, _, err = n.Append(dstDatBackend, sb.Version); err != nil {
return fmt.Errorf("cannot append needle: %s", err)
}
delta := n.DiskSize(opts.version)
delta := n.DiskSize(version)
newOffset += delta
writeThrottler.MaybeSlowdown(delta)
glog.V(4).Infoln("saving key", n.Id, "volume offset", offset, "=>", newOffset, "data_size", n.Size)
@@ -546,14 +518,14 @@ func (v *Volume) copyDataBasedOnIndexFile(opts *CompactOptions) (err error) {
v.Id.String(), v.nm.ContentSize(), v.nm.DeletedSize(), dstDatSize)
}
}
err = newNm.SaveToIdx(opts.destIdxPath)
err = newNm.SaveToIdx(datIdxName)
if err != nil {
return err
}
indexFile, err := os.OpenFile(opts.destIdxPath, os.O_RDWR|os.O_CREATE, 0644)
indexFile, err := os.OpenFile(datIdxName, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
glog.Errorf("cannot open Volume Index %s: %v", opts.destIdxPath, err)
glog.Errorf("cannot open Volume Index %s: %v", datIdxName, err)
return err
}
defer func() {
+4 -4
View File
@@ -62,14 +62,14 @@ func TestMakeDiff(t *testing.T) {
}
func TestMemIndexCompaction(t *testing.T) {
testCompactionByIndex(t, NeedleMapInMemory)
testCompaction(t, NeedleMapInMemory)
}
func TestLDBIndexCompaction(t *testing.T) {
testCompactionByIndex(t, NeedleMapLevelDb)
testCompaction(t, NeedleMapLevelDb)
}
func testCompactionByIndex(t *testing.T, needleMapKind NeedleMapKind) {
func testCompaction(t *testing.T, needleMapKind NeedleMapKind) {
dir := t.TempDir()
v, err := NewVolume(dir, dir, "", 1, needleMapKind, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
@@ -87,7 +87,7 @@ func testCompactionByIndex(t *testing.T, needleMapKind NeedleMapKind) {
}
startTime := time.Now()
v.CompactByIndex(nil)
v.Compact2(0, 0, nil)
speed := float64(v.ContentSize()) / time.Now().Sub(startTime).Seconds()
t.Logf("compaction speed: %.2f bytes/s", speed)
+1 -1
View File
@@ -9,7 +9,7 @@ import (
var (
MAJOR_VERSION = int32(4)
MINOR_VERSION = int32(13)
MINOR_VERSION = int32(12)
VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION)
VERSION = util.SizeLimit + " " + VERSION_NUMBER
COMMIT = ""