mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-13 10:05:50 +00:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8500a3bc56 | ||
|
|
678c52e74f | ||
|
|
8fac0cdab8 | ||
|
|
09bfb28db9 | ||
|
|
3bd990d2ea | ||
|
|
5e040ba03d |
@@ -111,7 +111,7 @@ jobs:
|
||||
|
||||
# Wait for S3 API
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8333/healthz > /dev/null 2>&1; then
|
||||
if curl -s http://localhost:8333/ > /dev/null 2>&1; then
|
||||
echo "✓ S3 API is ready"
|
||||
break
|
||||
fi
|
||||
|
||||
@@ -461,8 +461,6 @@ jobs:
|
||||
export S3_ENDPOINT="http://localhost:8006"
|
||||
export S3_ACCESS_KEY="0555b35654ad1656d804"
|
||||
export S3_SECRET_KEY="h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q=="
|
||||
export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
|
||||
export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY"
|
||||
|
||||
# Run the specific test that is equivalent to AWS S3 tagging behavior
|
||||
make test-with-server || {
|
||||
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
# Verify service accessibility
|
||||
echo "=== Verifying Service Accessibility ==="
|
||||
curl -f http://localhost:8080/realms/master
|
||||
curl -s http://localhost:8333/healthz
|
||||
curl -s http://localhost:8333
|
||||
echo "✅ SeaweedFS S3 API is responding (IAM-protected endpoint)"
|
||||
|
||||
# Run Keycloak-specific tests
|
||||
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
set -e
|
||||
mkdir -p /tmp/data
|
||||
./weed -v=3 server -s3 -dir=/tmp/data -s3.config=../docker/compose/s3.json -master.peers=none > weed.log 2>&1 &
|
||||
until curl -s http://localhost:8333/healthz > /dev/null; do sleep 1; done
|
||||
until curl -s http://localhost:8333/ > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Setup Caddy
|
||||
run: |
|
||||
@@ -54,7 +54,7 @@ jobs:
|
||||
- name: Start Caddy
|
||||
run: |
|
||||
./caddy start
|
||||
until curl -fsS --insecure https://localhost:8443/healthz > /dev/null; do sleep 1; done
|
||||
until curl -fsS --insecure https://localhost:8443 > /dev/null; do sleep 1; done
|
||||
|
||||
- name: Create Bucket
|
||||
run: |
|
||||
|
||||
@@ -46,7 +46,7 @@ require (
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/klauspost/compress v1.18.4
|
||||
github.com/klauspost/compress v1.18.3
|
||||
github.com/klauspost/reedsolomon v1.13.0
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/linxGnu/grocksdb v1.10.7
|
||||
@@ -130,7 +130,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.95.0
|
||||
github.com/cognusion/imaging v1.0.2
|
||||
github.com/fluent/fluent-logger-golang v1.10.1
|
||||
github.com/getsentry/sentry-go v0.42.0
|
||||
github.com/getsentry/sentry-go v0.40.0
|
||||
github.com/gin-contrib/sessions v1.0.4
|
||||
github.com/gin-gonic/gin v1.11.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.12
|
||||
@@ -142,7 +142,7 @@ require (
|
||||
github.com/jhump/protoreflect v1.18.0
|
||||
github.com/lib/pq v1.11.1
|
||||
github.com/linkedin/goavro/v2 v2.14.1
|
||||
github.com/mattn/go-sqlite3 v1.14.34
|
||||
github.com/mattn/go-sqlite3 v1.14.33
|
||||
github.com/minio/crc64nvme v1.1.1
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/parquet-go/parquet-go v0.26.4
|
||||
@@ -150,7 +150,7 @@ require (
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
github.com/rclone/rclone v1.72.1
|
||||
github.com/rdleal/intervalst v1.5.0
|
||||
github.com/redis/go-redis/v9 v9.18.0
|
||||
github.com/redis/go-redis/v9 v9.17.2
|
||||
github.com/schollz/progressbar/v3 v3.19.0
|
||||
github.com/seaweedfs/go-fuse/v2 v2.9.1
|
||||
github.com/shirou/gopsutil/v4 v4.26.1
|
||||
@@ -297,7 +297,7 @@ require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.1.0 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
|
||||
@@ -569,8 +569,8 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGlu
|
||||
github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3 h1:sxgSqOB9CDToiaVFpxuvb5wGgGqWa3lCShcm5o0n3bE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.5.3/go.mod h1:XdED8i399lEVblYHTZM8eXaP07gv4Z58IL6ueMlVlrg=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
|
||||
@@ -1053,8 +1053,8 @@ github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj
|
||||
github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/geoffgarside/ber v1.2.0 h1:/loowoRcs/MWLYmGX9QtIAbA+V/FrnVLsMMPhwiRm64=
|
||||
github.com/geoffgarside/ber v1.2.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8=
|
||||
github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
|
||||
github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo=
|
||||
github.com/getsentry/sentry-go v0.40.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
|
||||
github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
|
||||
@@ -1487,8 +1487,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
|
||||
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
|
||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
|
||||
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
|
||||
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
|
||||
github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.0.10/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
|
||||
@@ -1564,8 +1564,8 @@ github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhg
|
||||
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
|
||||
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
|
||||
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mattn/go-sqlite3 v1.14.33 h1:A5blZ5ulQo2AtayQ9/limgHEkFreKj1Dv226a1K73s0=
|
||||
github.com/mattn/go-sqlite3 v1.14.33/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
@@ -1814,8 +1814,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rdleal/intervalst v1.5.0 h1:SEB9bCFz5IqD1yhfH1Wv8IBnY/JQxDplwkxHjT6hamU=
|
||||
github.com/rdleal/intervalst v1.5.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/redis/go-redis/v9 v9.17.2 h1:P2EGsA4qVIM3Pp+aPocCJ7DguDHhqrXNhVcEp4ViluI=
|
||||
github.com/redis/go-redis/v9 v9.17.2/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
|
||||
github.com/redis/rueidis v1.0.69 h1:WlUefRhuDekji5LsD387ys3UCJtSFeBVf0e5yI0B8b4=
|
||||
github.com/redis/rueidis v1.0.69/go.mod h1:Lkhr2QTgcoYBhxARU7kJRO8SyVlgUuEkcJO1Y8MCluA=
|
||||
github.com/redis/rueidis/rueidiscompat v1.0.69 h1:IWVYY9lXdjNO3do2VpJT7aDFi8zbCUuQxZB6E2Grahs=
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.13"
|
||||
appVersion: "4.12"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.0.413
|
||||
version: 4.0.412
|
||||
|
||||
@@ -19,6 +19,9 @@ services:
|
||||
start_period: 10s
|
||||
|
||||
seaweedfs-volume:
|
||||
build:
|
||||
context: ../../../docker
|
||||
dockerfile: Dockerfile.local
|
||||
image: seaweedfs:local
|
||||
container_name: seaweedfs-spark-volume
|
||||
ports:
|
||||
@@ -40,6 +43,9 @@ services:
|
||||
start_period: 10s
|
||||
|
||||
seaweedfs-filer:
|
||||
build:
|
||||
context: ../../../docker
|
||||
dockerfile: Dockerfile.local
|
||||
image: seaweedfs:local
|
||||
container_name: seaweedfs-spark-filer
|
||||
ports:
|
||||
|
||||
@@ -116,7 +116,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
<artifactId>avro</artifactId>
|
||||
<version>1.11.5</version>
|
||||
<version>1.11.4</version>
|
||||
</dependency>
|
||||
|
||||
<!-- Apache ZooKeeper - Fix CVEs -->
|
||||
|
||||
@@ -12,7 +12,6 @@ FILER_PORT := 8888
|
||||
TEST_TIMEOUT := 10m
|
||||
TEST_PATTERN := TestCORS
|
||||
SERVER_DIR := test-mini-data
|
||||
S3_CONFIG := s3_test_config.json
|
||||
|
||||
# Default target
|
||||
help:
|
||||
@@ -81,15 +80,13 @@ start-server: check-deps
|
||||
@echo "🔍 DEBUG: Creating volume directory..."
|
||||
@mkdir -p $(SERVER_DIR)
|
||||
@echo "🔍 DEBUG: Launching SeaweedFS S3 server in background..."
|
||||
@echo "🔍 DEBUG: Command: AWS_ACCESS_KEY_ID=some_access_key1 AWS_SECRET_ACCESS_KEY=some_secret_key1 $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
|
||||
@env AWS_ACCESS_KEY_ID=some_access_key1 \
|
||||
AWS_SECRET_ACCESS_KEY=some_secret_key1 \
|
||||
$(WEED_BINARY) mini \
|
||||
@echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=$(SERVER_DIR) -s3.port=$(S3_PORT) -s3.config=$(S3_CONFIG)"
|
||||
@$(WEED_BINARY) mini \
|
||||
-dir=$(SERVER_DIR) \
|
||||
-s3.port=$(S3_PORT) \
|
||||
-s3.config=$(S3_CONFIG) \
|
||||
> weed-test.log 2>&1 & \
|
||||
echo $$! > weed-server.pid
|
||||
echo $$! > weed-test.pid
|
||||
|
||||
@echo "Waiting for S3 server to be ready..."
|
||||
@for i in $$(seq 1 30); do \
|
||||
@@ -100,7 +97,7 @@ start-server: check-deps
|
||||
sleep 1; \
|
||||
done; \
|
||||
echo "S3 server failed to start"; \
|
||||
exit 1
|
||||
exit 1 > weed-server.pid
|
||||
@echo "🔍 DEBUG: Server PID: $$(cat weed-server.pid 2>/dev/null || echo 'PID file not found')"
|
||||
@echo "🔍 DEBUG: Checking if PID is still running..."
|
||||
@sleep 2
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "anonymous",
|
||||
"actions": [
|
||||
"Read",
|
||||
"List"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{
|
||||
"accessKey": "some_access_key1",
|
||||
"secretKey": "some_secret_key1"
|
||||
}
|
||||
],
|
||||
"actions": [
|
||||
"Admin",
|
||||
"Read",
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package example
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -21,8 +22,10 @@ func TestIAMOperations(t *testing.T) {
|
||||
// Set credentials before starting cluster
|
||||
accessKey := "testkey123"
|
||||
secretKey := "testsecret456"
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", accessKey)
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
|
||||
os.Setenv("AWS_ACCESS_KEY_ID", accessKey)
|
||||
os.Setenv("AWS_SECRET_ACCESS_KEY", secretKey)
|
||||
defer os.Unsetenv("AWS_ACCESS_KEY_ID")
|
||||
defer os.Unsetenv("AWS_SECRET_ACCESS_KEY")
|
||||
|
||||
// Create and start test cluster
|
||||
cluster, err := startMiniCluster(t)
|
||||
|
||||
@@ -146,14 +146,6 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
return nil, fmt.Errorf("failed to create security.toml: %v", err)
|
||||
}
|
||||
|
||||
// Set environment variables for admin credentials safely for this test
|
||||
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
|
||||
}
|
||||
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
|
||||
}
|
||||
|
||||
// Start weed mini in a goroutine by calling the command directly
|
||||
cluster.wg.Add(1)
|
||||
go func() {
|
||||
|
||||
@@ -216,14 +216,6 @@ enabled = true
|
||||
err = os.WriteFile(credentialToml, []byte(credentialConfig), 0644)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Set environment variables for admin credentials safely for this test
|
||||
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
|
||||
}
|
||||
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
|
||||
}
|
||||
|
||||
cluster.wg.Add(1)
|
||||
go func() {
|
||||
defer cluster.wg.Done()
|
||||
|
||||
@@ -77,14 +77,14 @@ func TestObjectLockValidation(t *testing.T) {
|
||||
require.NoError(t, err, "Setting Object Lock retention should succeed")
|
||||
t.Log(" Object Lock retention applied successfully")
|
||||
|
||||
// Verify retention allows simple DELETE (creates delete marker) but blocks version deletion
|
||||
// AWS S3 behavior: Simple DELETE (without version ID) is ALWAYS allowed and creates delete marker
|
||||
// Verify retention blocks simple DELETE (COMPLIANCE mode is strict WORM)
|
||||
// AWS S3 behavior: Simple DELETE (without version ID) is blocked by COMPLIANCE retention
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker (AWS S3 behavior)")
|
||||
t.Log(" Simple DELETE succeeded (creates delete marker - correct AWS behavior)")
|
||||
require.Error(t, err, "Simple DELETE should be blocked by COMPLIANCE retention (AWS S3 behavior)")
|
||||
t.Log(" Simple DELETE correctly blocked by COMPLIANCE retention")
|
||||
|
||||
// Now verify that DELETE with version ID is properly blocked by retention
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
|
||||
@@ -329,12 +329,12 @@ func TestRetentionModeCompliance(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockRetentionModeCompliance, retentionResp.Retention.Mode)
|
||||
|
||||
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
|
||||
// Try simple DELETE - should fail for COMPLIANCE mode (strict WORM)
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
|
||||
require.Error(t, err, "Simple DELETE should be blocked by COMPLIANCE retention")
|
||||
|
||||
// Try DELETE with version ID - should fail for COMPLIANCE mode
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
@@ -388,12 +388,12 @@ func TestLegalHoldWorkflow(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, types.ObjectLockLegalHoldStatusOn, legalHoldResp.LegalHold.Status)
|
||||
|
||||
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
|
||||
// Try simple DELETE - should fail due to legal hold
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
|
||||
require.Error(t, err, "Simple DELETE should be blocked by legal hold")
|
||||
|
||||
// Try DELETE with version ID - should fail due to legal hold
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
@@ -591,12 +591,12 @@ func TestRetentionAndLegalHoldCombination(t *testing.T) {
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
|
||||
// Try simple DELETE - should fail due to legal hold + COMPLIANCE retention
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
|
||||
require.Error(t, err, "Simple DELETE should be blocked by legal hold")
|
||||
|
||||
// Try DELETE with version ID and bypass - should still fail due to legal hold
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
@@ -607,7 +607,7 @@ func TestRetentionAndLegalHoldCombination(t *testing.T) {
|
||||
})
|
||||
require.Error(t, err, "Legal hold should prevent deletion even with governance bypass")
|
||||
|
||||
// Remove legal hold (must specify version ID since latest version is now delete marker)
|
||||
// Remove legal hold (must specify version ID)
|
||||
_, err = client.PutObjectLegalHold(context.TODO(), &s3.PutObjectLegalHoldInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
|
||||
@@ -42,12 +42,12 @@ func TestWORMRetentionIntegration(t *testing.T) {
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
|
||||
// Try simple DELETE - should fail due to GOVERNANCE retention
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
|
||||
require.Error(t, err, "Simple DELETE should be blocked by GOVERNANCE retention")
|
||||
|
||||
// Try DELETE with version ID - should fail due to GOVERNANCE retention
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
@@ -325,12 +325,12 @@ func TestRetentionWithMultipartUpload(t *testing.T) {
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Try simple DELETE - should succeed and create delete marker (AWS S3 behavior)
|
||||
// Try simple DELETE - should fail due to GOVERNANCE retention
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String(key),
|
||||
})
|
||||
require.NoError(t, err, "Simple DELETE should succeed and create delete marker")
|
||||
require.Error(t, err, "Simple DELETE should be blocked by GOVERNANCE retention")
|
||||
|
||||
// Try DELETE with version ID - should fail due to GOVERNANCE retention
|
||||
_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
|
||||
|
||||
@@ -511,10 +511,8 @@ func createTableBucket(t *testing.T, env *TestEnvironment, bucketName string) {
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Logf("Create table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
|
||||
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusConflict {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("Failed to create table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
|
||||
}
|
||||
t.Logf("Created table bucket %s", bucketName)
|
||||
|
||||
@@ -56,6 +56,8 @@ func TestPyIcebergRestCatalog(t *testing.T) {
|
||||
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-e", fmt.Sprintf("AWS_ACCESS_KEY_ID=%s", "test"),
|
||||
"-e", fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%s", "test"),
|
||||
"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
|
||||
"-v", fmt.Sprintf("%s:/app:ro", testDir),
|
||||
"iceberg-rest-test",
|
||||
@@ -76,69 +78,3 @@ func TestPyIcebergRestCatalog(t *testing.T) {
|
||||
t.Errorf("PyIceberg test failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPyIcebergRestCatalogAuthenticated tests the Iceberg REST Catalog using PyIceberg with authentication.
|
||||
// This test uses the default admin credentials that SeaweedFS creates on startup.
|
||||
func TestPyIcebergRestCatalogAuthenticated(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
env := NewTestEnvironment(t)
|
||||
defer env.Cleanup(t)
|
||||
|
||||
if !env.dockerAvailable {
|
||||
t.Skip("Docker not available, skipping PyIceberg integration test")
|
||||
}
|
||||
|
||||
// Use default admin credentials
|
||||
testAccessKey := "admin"
|
||||
testSecretKey := "admin"
|
||||
|
||||
// Start SeaweedFS (it will use default admin credentials from environment if set)
|
||||
env.StartSeaweedFS(t)
|
||||
|
||||
// Create the test bucket first (using unauthenticated request, which works with DefaultAllow)
|
||||
bucketName := "pyiceberg-auth-test"
|
||||
createTableBucket(t, env, bucketName)
|
||||
|
||||
// Build the test working directory path
|
||||
testDir := filepath.Join(env.seaweedDir, "test", "s3tables", "catalog")
|
||||
|
||||
// Run PyIceberg test using Docker with authentication
|
||||
catalogURL := fmt.Sprintf("http://host.docker.internal:%d", env.icebergPort)
|
||||
s3Endpoint := fmt.Sprintf("http://host.docker.internal:%d", env.s3Port)
|
||||
warehouse := fmt.Sprintf("s3://%s/", bucketName)
|
||||
|
||||
// Build the test image first for faster repeated runs
|
||||
buildCmd := exec.Command("docker", "build", "-t", "iceberg-rest-test", "-f", "Dockerfile.pyiceberg", ".")
|
||||
buildCmd.Dir = testDir
|
||||
if out, err := buildCmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("Failed to build test image: %v\n%s", err, string(out))
|
||||
}
|
||||
|
||||
cmd := exec.Command("docker", "run", "--rm",
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-e", fmt.Sprintf("AWS_ENDPOINT_URL=%s", s3Endpoint),
|
||||
"-v", fmt.Sprintf("%s:/app:ro", testDir),
|
||||
"iceberg-rest-test",
|
||||
"python3", "/app/test_rest_catalog_auth.py",
|
||||
"--catalog-url", catalogURL,
|
||||
"--warehouse", warehouse,
|
||||
"--prefix", bucketName,
|
||||
"--access-key", testAccessKey,
|
||||
"--secret-key", testSecretKey,
|
||||
)
|
||||
cmd.Dir = testDir
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
t.Logf("Running PyIceberg REST catalog test with authentication...")
|
||||
t.Logf(" Catalog URL: %s", catalogURL)
|
||||
t.Logf(" Warehouse: %s", warehouse)
|
||||
t.Logf(" Access Key: %s", testAccessKey)
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
t.Errorf("PyIceberg authenticated test failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// verifyTableBucketMetadata verifies that a table bucket was created with proper metadata
|
||||
func verifyTableBucketMetadata(t *testing.T, env *TestEnvironment, bucketName string) {
|
||||
t.Helper()
|
||||
|
||||
// Use S3Tables REST API to get the bucket
|
||||
endpoint := fmt.Sprintf("http://localhost:%d/buckets/%s", env.s3Port, bucketName)
|
||||
|
||||
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-amz-json-1.1")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get table bucket %s: %v", bucketName, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
t.Logf("Get table bucket %s response: status=%d, body=%s", bucketName, resp.StatusCode, string(body))
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("Failed to get table bucket %s, status %d: %s", bucketName, resp.StatusCode, body)
|
||||
}
|
||||
t.Logf("Verified table bucket %s exists with metadata", bucketName)
|
||||
}
|
||||
@@ -201,7 +201,6 @@ def main():
|
||||
"uri": args.catalog_url,
|
||||
"warehouse": args.warehouse,
|
||||
"prefix": args.prefix,
|
||||
"s3.anonymous": "true", # Disable AWS request signing for unauthenticated access
|
||||
}
|
||||
)
|
||||
print(f"Successfully connected to catalog on attempt {attempt + 1}")
|
||||
|
||||
@@ -1,223 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Iceberg REST Catalog Compatibility Test for SeaweedFS (Authenticated)
|
||||
|
||||
This script tests the Iceberg REST Catalog API compatibility with authentication.
|
||||
|
||||
Usage:
|
||||
python3 test_rest_catalog_auth.py --catalog-url http://localhost:8182 \\
|
||||
--access-key admin --secret-key admin
|
||||
|
||||
Requirements:
|
||||
pip install pyiceberg[s3fs]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pyiceberg.catalog import load_catalog
|
||||
from pyiceberg.schema import Schema
|
||||
from pyiceberg.types import (
|
||||
IntegerType,
|
||||
LongType,
|
||||
StringType,
|
||||
NestedField,
|
||||
)
|
||||
from pyiceberg.exceptions import (
|
||||
NamespaceAlreadyExistsError,
|
||||
NoSuchNamespaceError,
|
||||
TableAlreadyExistsError,
|
||||
NoSuchTableError,
|
||||
)
|
||||
|
||||
|
||||
def test_config_endpoint(catalog):
|
||||
"""Test that the catalog config endpoint returns valid configuration."""
|
||||
print("Testing /v1/config endpoint...")
|
||||
# The catalog is already loaded which means config endpoint worked
|
||||
print(" /v1/config endpoint working")
|
||||
return True
|
||||
|
||||
|
||||
def test_namespace_operations(catalog, prefix):
|
||||
"""Test namespace CRUD operations."""
|
||||
print("Testing namespace operations...")
|
||||
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
|
||||
|
||||
# List initial namespaces
|
||||
namespaces = catalog.list_namespaces()
|
||||
print(f" Initial namespaces: {namespaces}")
|
||||
|
||||
# Create namespace
|
||||
try:
|
||||
catalog.create_namespace(namespace)
|
||||
print(f" Created namespace: {namespace}")
|
||||
except NamespaceAlreadyExistsError:
|
||||
print(f" ! Namespace already exists: {namespace}")
|
||||
|
||||
# List namespaces (should include our new one)
|
||||
namespaces = catalog.list_namespaces()
|
||||
if namespace in namespaces:
|
||||
print(" Namespace appears in list")
|
||||
else:
|
||||
print(f" Namespace not found in list: {namespaces}")
|
||||
return False
|
||||
|
||||
# Get namespace properties
|
||||
try:
|
||||
props = catalog.load_namespace_properties(namespace)
|
||||
print(f" Loaded namespace properties: {props}")
|
||||
except NoSuchNamespaceError:
|
||||
print(f" Failed to load namespace properties")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def test_table_operations(catalog, prefix):
|
||||
"""Test table CRUD operations."""
|
||||
print("Testing table operations...")
|
||||
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
|
||||
table_name = "auth_test_table"
|
||||
table_id = namespace + (table_name,)
|
||||
|
||||
# Define a simple schema
|
||||
schema = Schema(
|
||||
NestedField(field_id=1, name="id", field_type=LongType(), required=True),
|
||||
NestedField(field_id=2, name="name", field_type=StringType(), required=False),
|
||||
NestedField(field_id=3, name="age", field_type=IntegerType(), required=False),
|
||||
)
|
||||
|
||||
# Create table
|
||||
try:
|
||||
table = catalog.create_table(
|
||||
identifier=table_id,
|
||||
schema=schema,
|
||||
)
|
||||
print(f" Created table: {table_id}")
|
||||
except TableAlreadyExistsError:
|
||||
print(f" ! Table already exists: {table_id}")
|
||||
_ = catalog.load_table(table_id)
|
||||
|
||||
# List tables
|
||||
tables = catalog.list_tables(namespace)
|
||||
if table_name in [t[1] for t in tables]:
|
||||
print(" Table appears in list")
|
||||
else:
|
||||
print(f" Table not found in list: {tables}")
|
||||
return False
|
||||
|
||||
# Load table
|
||||
try:
|
||||
loaded_table = catalog.load_table(table_id)
|
||||
print(f" Loaded table: {loaded_table.name()}")
|
||||
print(f" Schema: {loaded_table.schema()}")
|
||||
print(f" Location: {loaded_table.location()}")
|
||||
except NoSuchTableError:
|
||||
print(f" Failed to load table")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def test_cleanup(catalog, prefix):
|
||||
"""Test table and namespace deletion."""
|
||||
print("Testing cleanup operations...")
|
||||
namespace = (f"{prefix.replace('-', '_')}_auth_test_ns",)
|
||||
table_id = namespace + ("auth_test_table",)
|
||||
|
||||
# Drop table
|
||||
try:
|
||||
catalog.drop_table(table_id)
|
||||
print(f" Dropped table: {table_id}")
|
||||
except NoSuchTableError:
|
||||
print(f" ! Table already deleted: {table_id}")
|
||||
|
||||
# Drop namespace
|
||||
try:
|
||||
catalog.drop_namespace(namespace)
|
||||
print(f" Dropped namespace: {namespace}")
|
||||
except NoSuchNamespaceError:
|
||||
print(f" ! Namespace already deleted: {namespace}")
|
||||
except Exception as e:
|
||||
print(f" ? Namespace drop error (may be expected): {e}")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Test Iceberg REST Catalog with authentication")
|
||||
parser.add_argument("--catalog-url", required=True, help="Iceberg REST Catalog URL")
|
||||
parser.add_argument("--warehouse", default="s3://iceberg-test/", help="Warehouse location")
|
||||
parser.add_argument("--prefix", required=True, help="Table bucket prefix")
|
||||
parser.add_argument("--access-key", required=True, help="AWS Access Key ID")
|
||||
parser.add_argument("--secret-key", required=True, help="AWS Secret Access Key")
|
||||
parser.add_argument("--skip-cleanup", action="store_true", help="Skip cleanup at the end")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Connecting to Iceberg REST Catalog at: {args.catalog_url}")
|
||||
print(f"Warehouse: {args.warehouse}")
|
||||
print(f"Prefix: {args.prefix}")
|
||||
print(f"Using authenticated access with key: {args.access_key}")
|
||||
print()
|
||||
|
||||
# Load the REST catalog with authentication
|
||||
import time
|
||||
max_retries = 10
|
||||
catalog = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
catalog = load_catalog(
|
||||
"rest",
|
||||
**{
|
||||
"type": "rest",
|
||||
"uri": args.catalog_url,
|
||||
"warehouse": args.warehouse,
|
||||
"prefix": args.prefix,
|
||||
"s3.access-key-id": args.access_key,
|
||||
"s3.secret-access-key": args.secret_key,
|
||||
}
|
||||
)
|
||||
print(f"Successfully connected to catalog on attempt {attempt + 1}")
|
||||
break
|
||||
except Exception as e:
|
||||
if attempt < max_retries - 1:
|
||||
print(f" Attempt {attempt + 1} failed, retrying in 2s... ({e})")
|
||||
time.sleep(2)
|
||||
else:
|
||||
print(f" All {max_retries} attempts failed.")
|
||||
raise e
|
||||
|
||||
# Run tests
|
||||
tests = [
|
||||
("Config Endpoint", lambda: test_config_endpoint(catalog)),
|
||||
("Namespace Operations", lambda: test_namespace_operations(catalog, args.prefix)),
|
||||
("Table Operations", lambda: test_table_operations(catalog, args.prefix)),
|
||||
]
|
||||
|
||||
if not args.skip_cleanup:
|
||||
tests.append(("Cleanup", lambda: test_cleanup(catalog, args.prefix)))
|
||||
|
||||
passed = 0
|
||||
failed = 0
|
||||
|
||||
for name, test_fn in tests:
|
||||
print(f"\n{'='*50}")
|
||||
try:
|
||||
if test_fn():
|
||||
passed += 1
|
||||
print(f"PASSED: {name}")
|
||||
else:
|
||||
failed += 1
|
||||
print(f"FAILED: {name}")
|
||||
except Exception as e:
|
||||
failed += 1
|
||||
print(f"ERROR in {name}: {e}")
|
||||
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Results: {passed} passed, {failed} failed")
|
||||
|
||||
return 0 if failed == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -556,14 +556,6 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) {
|
||||
return nil, fmt.Errorf("failed to create security.toml: %v", err)
|
||||
}
|
||||
|
||||
// Set environment variables for admin credentials safely for this test
|
||||
if os.Getenv("AWS_ACCESS_KEY_ID") == "" {
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "admin")
|
||||
}
|
||||
if os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "admin")
|
||||
}
|
||||
|
||||
// Start weed mini in a goroutine by calling the command directly
|
||||
cluster.wg.Add(1)
|
||||
go func() {
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
package maintenance
|
||||
|
||||
const DefaultMasterMaintenanceScripts = `
|
||||
lock
|
||||
ec.encode -fullPercent=95 -quietFor=1h
|
||||
ec.rebuild -apply
|
||||
ec.balance -apply
|
||||
fs.log.purge -daysAgo=7
|
||||
volume.deleteEmpty -quietFor=24h -apply
|
||||
volume.balance -apply
|
||||
volume.fix.replication -apply
|
||||
s3.clean.uploads -timeAgo=24h
|
||||
unlock
|
||||
`
|
||||
@@ -137,7 +137,7 @@ func backupFromLocation(volumeServer pb.ServerAddress, grpcDialOption grpc.DialO
|
||||
|
||||
// Handle compaction if needed
|
||||
if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
|
||||
if err = v.CompactByIndex(nil); err != nil {
|
||||
if err = v.Compact2(0, 0, nil); err != nil {
|
||||
v.Close()
|
||||
return fmt.Errorf("compacting volume: %w", err), false
|
||||
}
|
||||
|
||||
+9
-20
@@ -1,8 +1,6 @@
|
||||
package command
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
@@ -20,9 +18,8 @@ var cmdCompact = &Command{
|
||||
The compacted .dat file is stored as .cpd file.
|
||||
The compacted .idx file is stored as .cpx file.
|
||||
|
||||
Supports two compaction methods:
|
||||
* data: compacts based on the .dat file, works if .idx file is corrupted.
|
||||
* index: compacts based on the .idx file, works if deletion happened but not written to .dat files.
|
||||
For method=0, it compacts based on the .dat file, works if .idx file is corrupted.
|
||||
For method=1, it compacts based on the .idx file, works if deletion happened but not written to .dat files.
|
||||
|
||||
`,
|
||||
}
|
||||
@@ -31,7 +28,7 @@ var (
|
||||
compactVolumePath = cmdCompact.Flag.String("dir", ".", "data directory to store files")
|
||||
compactVolumeCollection = cmdCompact.Flag.String("collection", "", "volume collection name")
|
||||
compactVolumeId = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
|
||||
compactMethod = cmdCompact.Flag.String("method", "data", "option to choose which compact method (data/index)")
|
||||
compactMethod = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 (default) or 1.")
|
||||
compactVolumePreallocate = cmdCompact.Flag.Int64("preallocateMB", 0, "preallocate volume disk space")
|
||||
)
|
||||
|
||||
@@ -41,29 +38,21 @@ func runCompact(cmd *Command, args []string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
preallocateBytes := *compactVolumePreallocate * (1 << 20)
|
||||
preallocate := *compactVolumePreallocate * (1 << 20)
|
||||
|
||||
vid := needle.VolumeId(*compactVolumeId)
|
||||
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocateBytes, needle.GetCurrentVersion(), 0, 0)
|
||||
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, needle.GetCurrentVersion(), 0, 0)
|
||||
if err != nil {
|
||||
glog.Fatalf("Load Volume [ERROR] %s\n", err)
|
||||
}
|
||||
|
||||
opts := &storage.CompactOptions{
|
||||
PreallocateBytes: preallocateBytes,
|
||||
MaxBytesPerSecond: 0, // unlimited
|
||||
}
|
||||
switch strings.ToLower(*compactMethod) {
|
||||
case "data":
|
||||
if err = v.CompactByVolumeData(opts); err != nil {
|
||||
if *compactMethod == 0 {
|
||||
if err = v.Compact(preallocate, 0); err != nil {
|
||||
glog.Fatalf("Compact Volume [ERROR] %s\n", err)
|
||||
}
|
||||
case "index":
|
||||
if err = v.CompactByIndex(opts); err != nil {
|
||||
} else {
|
||||
if err = v.Compact2(preallocate, 0, nil); err != nil {
|
||||
glog.Fatalf("Compact Volume [ERROR] %s\n", err)
|
||||
}
|
||||
default:
|
||||
glog.Fatalf("unsupported compaction method %q", *compactMethod)
|
||||
}
|
||||
|
||||
return true
|
||||
|
||||
+4
-14
@@ -355,9 +355,6 @@ func isFlagPassed(name string) bool {
|
||||
|
||||
// isPortOpenOnIP checks if a port is available for binding on a specific IP address
|
||||
func isPortOpenOnIP(ip string, port int) bool {
|
||||
if port <= 0 || port > 65535 {
|
||||
return false
|
||||
}
|
||||
listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", ip, port))
|
||||
if err != nil {
|
||||
return false
|
||||
@@ -369,9 +366,6 @@ func isPortOpenOnIP(ip string, port int) bool {
|
||||
// isPortAvailable checks if a port is available on any interface
|
||||
// This is more comprehensive than checking a single IP
|
||||
func isPortAvailable(port int) bool {
|
||||
if port <= 0 || port > 65535 {
|
||||
return false
|
||||
}
|
||||
// Try to listen on all interfaces (0.0.0.0)
|
||||
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
|
||||
if err != nil {
|
||||
@@ -387,10 +381,6 @@ func isPortAvailable(port int) bool {
|
||||
func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPorts map[int]bool) int {
|
||||
for i := 0; i < maxAttempts; i++ {
|
||||
port := startPort + i
|
||||
if port > 65535 {
|
||||
// Wrap around to a lower range if we exceed 65535
|
||||
port = 10000 + (port % 65535)
|
||||
}
|
||||
// Skip ports reserved for gRPC calculation
|
||||
if reservedPorts[port] {
|
||||
continue
|
||||
@@ -408,15 +398,15 @@ func findAvailablePortOnIP(ip string, startPort int, maxAttempts int, reservedPo
|
||||
// If the port is not available, it finds the next available port and updates the pointer
|
||||
// The reservedPorts map contains ports that should not be allocated (for gRPC collision avoidance)
|
||||
func ensurePortAvailableOnIP(portPtr *int, serviceName string, ip string, reservedPorts map[int]bool, flagName string) error {
|
||||
// Check if this port was explicitly specified by the user (from CLI, before config file was applied)
|
||||
isExplicitPort := explicitPortFlags[flagName]
|
||||
|
||||
if *portPtr == 0 {
|
||||
if portPtr == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
original := *portPtr
|
||||
|
||||
// Check if this port was explicitly specified by the user (from CLI, before config file was applied)
|
||||
isExplicitPort := explicitPortFlags[flagName]
|
||||
|
||||
// Skip if this port is reserved for gRPC calculation
|
||||
if reservedPorts[original] {
|
||||
if isExplicitPort {
|
||||
|
||||
@@ -1,15 +1,6 @@
|
||||
package scaffold
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Master = strings.ReplaceAll(masterTemplate, "{{DEFAULT_MAINTENANCE_SCRIPTS}}", maintenance.DefaultMasterMaintenanceScripts)
|
||||
}
|
||||
import _ "embed"
|
||||
|
||||
//go:embed filer.toml
|
||||
var Filer string
|
||||
@@ -24,7 +15,6 @@ var Replication string
|
||||
var Security string
|
||||
|
||||
//go:embed master.toml
|
||||
var masterTemplate string
|
||||
var Master string
|
||||
|
||||
//go:embed shell.toml
|
||||
|
||||
@@ -6,7 +6,17 @@
|
||||
|
||||
[master.maintenance]
|
||||
# periodically run these scripts are the same as running them from 'weed shell'
|
||||
scripts = """{{DEFAULT_MAINTENANCE_SCRIPTS}}"""
|
||||
scripts = """
|
||||
lock
|
||||
ec.encode -fullPercent=95 -quietFor=1h
|
||||
ec.rebuild -apply
|
||||
ec.balance -apply
|
||||
volume.deleteEmpty -quietFor=24h -apply
|
||||
volume.balance -apply
|
||||
volume.fix.replication -apply
|
||||
s3.clean.uploads -timeAgo=24h
|
||||
unlock
|
||||
"""
|
||||
sleep_minutes = 17 # sleep minutes between each script execution
|
||||
|
||||
|
||||
|
||||
@@ -43,7 +43,6 @@ expires_after_seconds = 10 # seconds
|
||||
# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
|
||||
# - f.e. the S3 API Shim generates the JWT
|
||||
# - the Filer server validates the JWT on writing
|
||||
# NOTE: This key is ALSO used as a fallback signing key for S3 STS if s3.iam.config does not specify a signingKey.
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
[jwt.filer_signing]
|
||||
key = ""
|
||||
|
||||
@@ -654,14 +654,6 @@ func (m *IAMManager) GetSTSService() *sts.STSService {
|
||||
return m.stsService
|
||||
}
|
||||
|
||||
// DefaultAllow returns whether the default effect is Allow
|
||||
func (m *IAMManager) DefaultAllow() bool {
|
||||
if !m.initialized || m.policyEngine == nil {
|
||||
return true // Default to true if not initialized
|
||||
}
|
||||
return m.policyEngine.DefaultAllow()
|
||||
}
|
||||
|
||||
// parseJWTTokenForTrustPolicy parses a JWT token to extract claims for trust policy evaluation
|
||||
func parseJWTTokenForTrustPolicy(tokenString string) (map[string]interface{}, error) {
|
||||
// Simple JWT parsing without verification (for trust policy context only)
|
||||
|
||||
@@ -324,14 +324,6 @@ func (e *PolicyEngine) IsInitialized() bool {
|
||||
return e.initialized
|
||||
}
|
||||
|
||||
// DefaultAllow returns whether the default effect is Allow
|
||||
func (e *PolicyEngine) DefaultAllow() bool {
|
||||
if e.config == nil {
|
||||
return true // Default to Allow if not configured
|
||||
}
|
||||
return e.config.DefaultEffect == string(EffectAllow)
|
||||
}
|
||||
|
||||
// AddPolicy adds a policy to the engine (filerAddress ignored for memory stores)
|
||||
func (e *PolicyEngine) AddPolicy(filerAddress string, name string, policy *PolicyDocument) error {
|
||||
if !e.initialized {
|
||||
|
||||
@@ -270,9 +270,6 @@ func (s *STSService) Initialize(config *STSConfig) error {
|
||||
return fmt.Errorf(ErrConfigCannotBeNil)
|
||||
}
|
||||
|
||||
// Apply defaults before validation
|
||||
config.ApplyDefaults()
|
||||
|
||||
if err := s.validateConfig(config); err != nil {
|
||||
return fmt.Errorf("invalid STS configuration: %w", err)
|
||||
}
|
||||
@@ -291,21 +288,6 @@ func (s *STSService) Initialize(config *STSConfig) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ApplyDefaults applies default values to the STS configuration
|
||||
func (c *STSConfig) ApplyDefaults() {
|
||||
if c.TokenDuration.Duration <= 0 {
|
||||
c.TokenDuration.Duration = time.Duration(DefaultTokenDuration) * time.Second
|
||||
}
|
||||
|
||||
if c.MaxSessionLength.Duration <= 0 {
|
||||
c.MaxSessionLength.Duration = time.Duration(DefaultMaxSessionLength) * time.Second
|
||||
}
|
||||
|
||||
if c.Issuer == "" {
|
||||
c.Issuer = DefaultIssuer
|
||||
}
|
||||
}
|
||||
|
||||
// validateConfig validates the STS configuration
|
||||
func (s *STSService) validateConfig(config *STSConfig) error {
|
||||
if config.TokenDuration.Duration <= 0 {
|
||||
|
||||
@@ -75,34 +75,11 @@ func TestSTSServiceInitialization(t *testing.T) {
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, service.IsInitialized())
|
||||
|
||||
// Verify defaults if applicable
|
||||
if tt.config.Issuer == "" {
|
||||
assert.Equal(t, DefaultIssuer, service.Config.Issuer)
|
||||
}
|
||||
if tt.config.TokenDuration.Duration == 0 {
|
||||
assert.Equal(t, time.Duration(DefaultTokenDuration)*time.Second, service.Config.TokenDuration.Duration)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSTSServiceDefaults(t *testing.T) {
|
||||
service := NewSTSService()
|
||||
config := &STSConfig{
|
||||
SigningKey: []byte("test-signing-key"),
|
||||
// Missing duration and issuer
|
||||
}
|
||||
|
||||
err := service.Initialize(config)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, DefaultIssuer, config.Issuer)
|
||||
assert.Equal(t, time.Duration(DefaultTokenDuration)*time.Second, config.TokenDuration.Duration)
|
||||
assert.Equal(t, time.Duration(DefaultMaxSessionLength)*time.Second, config.MaxSessionLength.Duration)
|
||||
}
|
||||
|
||||
// TestAssumeRoleWithWebIdentity tests role assumption with OIDC tokens
|
||||
func TestAssumeRoleWithWebIdentity(t *testing.T) {
|
||||
service := setupTestSTSService(t)
|
||||
|
||||
@@ -89,9 +89,7 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli
|
||||
GrpcDialOption: option.GrpcDialOption,
|
||||
}
|
||||
|
||||
// Initialize FilerClient for IAM - explicit filers only (no discovery as FilerGroup unspecified)
|
||||
filerClient := wdclient.NewFilerClient(option.Filers, option.GrpcDialOption, "")
|
||||
iam := s3api.NewIdentityAccessManagementWithStore(&s3Option, filerClient, explicitStore)
|
||||
iam := s3api.NewIdentityAccessManagementWithStore(&s3Option, explicitStore)
|
||||
configure.credentialManager = iam.GetCredentialManager()
|
||||
|
||||
iamApiServer = &IamApiServer{
|
||||
@@ -102,13 +100,6 @@ func NewIamApiServerWithStore(router *mux.Router, option *IamServerOption, expli
|
||||
masterClient: masterClient,
|
||||
}
|
||||
|
||||
// Keep attempting to load configuration from filer now that we have a client
|
||||
go func() {
|
||||
if err := iam.LoadS3ApiConfigurationFromCredentialManager(); err != nil {
|
||||
glog.Warningf("Failed to load IAM config from credential manager after client update: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
iamApiServer.registerRouter(router)
|
||||
|
||||
return iamApiServer, nil
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/policy_engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
|
||||
// Import KMS providers to register them
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/kms/aws"
|
||||
@@ -55,7 +54,7 @@ type IdentityAccessManagement struct {
|
||||
domain string
|
||||
isAuthEnabled bool
|
||||
credentialManager *credential.CredentialManager
|
||||
filerClient *wdclient.FilerClient
|
||||
filerClient filer_pb.SeaweedFilerClient
|
||||
grpcDialOption grpc.DialOption
|
||||
|
||||
// IAM Integration for advanced features
|
||||
@@ -133,37 +132,15 @@ func (c *Credential) isCredentialExpired() bool {
|
||||
return c.Expiration > 0 && c.Expiration < time.Now().Unix()
|
||||
}
|
||||
|
||||
// NewIdentityAccessManagement creates a new IAM manager
|
||||
// SetFilerClient updates the filer client and its associated credential store
|
||||
func (iam *IdentityAccessManagement) SetFilerClient(filerClient *wdclient.FilerClient) {
|
||||
iam.m.Lock()
|
||||
iam.filerClient = filerClient
|
||||
iam.m.Unlock()
|
||||
|
||||
if iam.credentialManager == nil || filerClient == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Update credential store to use FilerClient's current filer for HA
|
||||
if store := iam.credentialManager.GetStore(); store != nil {
|
||||
if filerFuncSetter, ok := store.(interface {
|
||||
SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption)
|
||||
}); ok {
|
||||
filerFuncSetter.SetFilerAddressFunc(filerClient.GetCurrentFiler, iam.grpcDialOption)
|
||||
}
|
||||
}
|
||||
func NewIdentityAccessManagement(option *S3ApiServerOption) *IdentityAccessManagement {
|
||||
return NewIdentityAccessManagementWithStore(option, "")
|
||||
}
|
||||
|
||||
func NewIdentityAccessManagement(option *S3ApiServerOption, filerClient *wdclient.FilerClient) *IdentityAccessManagement {
|
||||
return NewIdentityAccessManagementWithStore(option, filerClient, "")
|
||||
}
|
||||
|
||||
func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient *wdclient.FilerClient, explicitStore string) *IdentityAccessManagement {
|
||||
func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, explicitStore string) *IdentityAccessManagement {
|
||||
iam := &IdentityAccessManagement{
|
||||
domain: option.DomainName,
|
||||
hashes: make(map[string]*sync.Pool),
|
||||
hashCounters: make(map[string]*int32),
|
||||
filerClient: filerClient,
|
||||
}
|
||||
|
||||
// Always initialize credential manager with fallback to defaults
|
||||
@@ -195,25 +172,6 @@ func NewIdentityAccessManagementWithStore(option *S3ApiServerOption, filerClient
|
||||
|
||||
iam.credentialManager = credentialManager
|
||||
iam.stopChan = make(chan struct{})
|
||||
iam.grpcDialOption = option.GrpcDialOption
|
||||
|
||||
// Initialize default anonymous identity
|
||||
// This ensures consistent behavior for anonymous access:
|
||||
// 1. In simple auth mode (no IAM integration):
|
||||
// - lookupAnonymous returns this identity
|
||||
// - VerifyActionPermission checks actions (which are empty) -> Denies access
|
||||
// - This preserves the secure-by-default behavior for simple auth
|
||||
// 2. In advanced IAM mode (with Policy Engine):
|
||||
// - lookupAnonymous returns this identity
|
||||
// - VerifyActionPermission proceeds to Policy Engine
|
||||
// - Policy Engine evaluates against policies (DefaultEffect=Allow if no config)
|
||||
// - This enables the flexible "Open by Default" for zero-config startup
|
||||
iam.identityAnonymous = &Identity{
|
||||
Name: "anonymous",
|
||||
Account: &AccountAnonymous,
|
||||
Actions: []Action{},
|
||||
IsStatic: true,
|
||||
}
|
||||
|
||||
// First, try to load configurations from file or filer
|
||||
startConfigFile := option.Config
|
||||
@@ -594,16 +552,6 @@ func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure anonymous identity exists
|
||||
if identityAnonymous == nil {
|
||||
identityAnonymous = &Identity{
|
||||
Name: "anonymous",
|
||||
Account: accounts[AccountAnonymous.Id],
|
||||
Actions: []Action{},
|
||||
IsStatic: true,
|
||||
}
|
||||
}
|
||||
|
||||
// atomically switch
|
||||
iam.identities = identities
|
||||
iam.identityAnonymous = identityAnonymous
|
||||
@@ -624,9 +572,6 @@ func (iam *IdentityAccessManagement) ReplaceS3ApiConfiguration(config *iam_pb.S3
|
||||
}
|
||||
}
|
||||
if !exists {
|
||||
if len(envIdent.Credentials) == 0 {
|
||||
continue
|
||||
}
|
||||
iam.identities = append(iam.identities, envIdent)
|
||||
iam.accessKeyIdent[envIdent.Credentials[0].AccessKey] = envIdent
|
||||
iam.nameToIdentity[envIdent.Name] = envIdent
|
||||
@@ -1047,8 +992,7 @@ func (iam *IdentityAccessManagement) LookupByAccessKey(accessKey string) (identi
|
||||
return iam.lookupByAccessKey(accessKey)
|
||||
}
|
||||
|
||||
// LookupAnonymous returns the anonymous identity if it exists
|
||||
func (iam *IdentityAccessManagement) LookupAnonymous() (identity *Identity, found bool) {
|
||||
func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, found bool) {
|
||||
iam.m.RLock()
|
||||
defer iam.m.RUnlock()
|
||||
if iam.identityAnonymous != nil {
|
||||
@@ -1168,9 +1112,6 @@ func (iam *IdentityAccessManagement) handleAuthResult(w http.ResponseWriter, r *
|
||||
// Wrapper to maintain backward compatibility
|
||||
func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action) (*Identity, s3err.ErrorCode) {
|
||||
identity, err, _ := iam.authRequestWithAuthType(r, action)
|
||||
if err != s3err.ErrNone {
|
||||
return nil, err
|
||||
}
|
||||
return identity, err
|
||||
}
|
||||
|
||||
@@ -1232,7 +1173,7 @@ func (iam *IdentityAccessManagement) authenticateRequestInternal(r *http.Request
|
||||
}
|
||||
case authTypeAnonymous:
|
||||
amzAuthType = "Anonymous"
|
||||
if identity, found = iam.LookupAnonymous(); !found {
|
||||
if identity, found = iam.lookupAnonymous(); !found {
|
||||
r.Header.Set(s3_constants.AmzAuthType, amzAuthType)
|
||||
return identity, s3err.ErrAccessDenied, reqAuthType
|
||||
}
|
||||
@@ -1271,8 +1212,8 @@ func (iam *IdentityAccessManagement) authRequestWithAuthType(r *http.Request, ac
|
||||
// through buckets and checking permissions for each. Skip the global check here.
|
||||
policyAllows := false
|
||||
|
||||
if action == s3_constants.ACTION_LIST && bucket == "" && identity.Name != s3_constants.AccountAnonymousId {
|
||||
// ListBuckets operation for authenticated users - authorization handled per-bucket in the handler
|
||||
if action == s3_constants.ACTION_LIST && bucket == "" {
|
||||
// ListBuckets operation - authorization handled per-bucket in the handler
|
||||
} else {
|
||||
// First check bucket policy if one exists
|
||||
// Bucket policies can grant or deny access to specific users/principals
|
||||
@@ -1366,8 +1307,8 @@ func (iam *IdentityAccessManagement) AuthSignatureOnly(r *http.Request) (*Identi
|
||||
return identity, s3err.ErrNotImplemented
|
||||
}
|
||||
case authTypeAnonymous:
|
||||
// Anonymous users can be authenticated, but authorization is handled separately
|
||||
return iam.identityAnonymous, s3err.ErrNone
|
||||
// Anonymous users cannot use IAM API
|
||||
return identity, s3err.ErrAccessDenied
|
||||
default:
|
||||
return identity, s3err.ErrNotImplemented
|
||||
}
|
||||
|
||||
@@ -450,7 +450,7 @@ func TestNewIdentityAccessManagementWithStoreEnvVars(t *testing.T) {
|
||||
option := &S3ApiServerOption{
|
||||
Config: "", // No config file - this should trigger environment variable fallback
|
||||
}
|
||||
iam := NewIdentityAccessManagementWithStore(option, nil, string(credential.StoreTypeMemory))
|
||||
iam := NewIdentityAccessManagementWithStore(option, string(credential.StoreTypeMemory))
|
||||
|
||||
if tt.expectEnvIdentity {
|
||||
// Should have exactly one identity from environment variables
|
||||
@@ -510,7 +510,7 @@ func TestConfigFileWithNoIdentitiesAllowsEnvVars(t *testing.T) {
|
||||
option := &S3ApiServerOption{
|
||||
Config: tmpFile.Name(),
|
||||
}
|
||||
iam := NewIdentityAccessManagementWithStore(option, nil, string(credential.StoreTypeMemory))
|
||||
iam := NewIdentityAccessManagementWithStore(option, string(credential.StoreTypeMemory))
|
||||
|
||||
// Should have exactly one identity from environment variables
|
||||
assert.Len(t, iam.identities, 1, "Should have exactly one identity from environment variables even when config file exists with no identities")
|
||||
@@ -762,7 +762,7 @@ func TestSignatureVerificationDoesNotCheckPermissions(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStaticIdentityProtection(t *testing.T) {
|
||||
iam := NewIdentityAccessManagement(&S3ApiServerOption{}, nil)
|
||||
iam := NewIdentityAccessManagement(&S3ApiServerOption{})
|
||||
|
||||
// Add a static identity
|
||||
staticIdent := &Identity{
|
||||
|
||||
@@ -66,7 +66,7 @@ func TestReproIssue7912(t *testing.T) {
|
||||
option := &S3ApiServerOption{
|
||||
Config: tmpFile.Name(),
|
||||
}
|
||||
iam := NewIdentityAccessManagementWithStore(option, nil, "memory")
|
||||
iam := NewIdentityAccessManagementWithStore(option, "memory")
|
||||
|
||||
assert.True(t, iam.isEnabled(), "Auth should be enabled")
|
||||
|
||||
|
||||
@@ -44,10 +44,6 @@ func (m *MockIAMIntegration) ValidateTrustPolicyForPrincipal(ctx context.Context
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockIAMIntegration) DefaultAllow() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// TestVerifyV4SignatureWithSTSIdentity tests that verifyV4Signature properly handles STS identities
|
||||
// by falling back to IAM authorization when shouldCheckPermissions is true
|
||||
func TestVerifyV4SignatureWithSTSIdentity(t *testing.T) {
|
||||
|
||||
@@ -22,7 +22,7 @@ func TestSTSIdentityPolicyNamesPopulation(t *testing.T) {
|
||||
stsService, config := setupTestSTSService(t)
|
||||
|
||||
// Create IAM with STS integration
|
||||
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
|
||||
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, "memory")
|
||||
s3iam := &S3IAMIntegration{
|
||||
stsService: stsService,
|
||||
}
|
||||
@@ -264,7 +264,7 @@ func TestValidateSTSSessionTokenIntegration(t *testing.T) {
|
||||
stsService, config := setupTestSTSService(t)
|
||||
|
||||
// Create IAM with STS integration
|
||||
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
|
||||
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, "memory")
|
||||
s3iam := &S3IAMIntegration{
|
||||
stsService: stsService,
|
||||
}
|
||||
@@ -311,7 +311,7 @@ func TestSTSIdentityClaimsPopulation(t *testing.T) {
|
||||
stsService, config := setupTestSTSService(t)
|
||||
|
||||
// Create IAM with STS integration
|
||||
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, "memory")
|
||||
iam := NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, "memory")
|
||||
s3iam := &S3IAMIntegration{
|
||||
stsService: stsService,
|
||||
}
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestLoadIAMManagerFromConfig_Defaults(t *testing.T) {
|
||||
// Create a temporary config file with minimal content (just policy)
|
||||
tmpDir := t.TempDir()
|
||||
configPath := filepath.Join(tmpDir, "iam_config.json")
|
||||
|
||||
configContent := `{
|
||||
"sts": {
|
||||
"providers": []
|
||||
},
|
||||
"policy": {
|
||||
"storeType": "memory",
|
||||
"defaultEffect": "Allow"
|
||||
}
|
||||
}`
|
||||
|
||||
err := os.WriteFile(configPath, []byte(configContent), 0644)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// dummy filer address provider
|
||||
filerProvider := func() string { return "localhost:8888" }
|
||||
defaultSigningKeyProvider := func() string { return "default-secure-signing-key" }
|
||||
|
||||
// Load the manager
|
||||
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, manager)
|
||||
}
|
||||
|
||||
func TestLoadIAMManagerFromConfig_Overrides(t *testing.T) {
|
||||
// Create a temporary config file with EXPLICIT values
|
||||
tmpDir := t.TempDir()
|
||||
configPath := filepath.Join(tmpDir, "iam_config_explicit.json")
|
||||
|
||||
configContent := `{
|
||||
"sts": {
|
||||
"tokenDuration": "2h",
|
||||
"maxSessionLength": "24h",
|
||||
"issuer": "custom-issuer",
|
||||
"signingKey": "ZXhwbGljaXQtc2lnbmluZy1rZXktMTIzNDU="
|
||||
},
|
||||
"policy": {
|
||||
"storeType": "memory",
|
||||
"defaultEffect": "Allow"
|
||||
}
|
||||
}`
|
||||
// Base64 encoded "explicit-signing-key-12345" is "ZXhwbGljaXQtc2lnbmluZy1rZXktMTIzNDU="
|
||||
|
||||
err := os.WriteFile(configPath, []byte(configContent), 0644)
|
||||
assert.NoError(t, err)
|
||||
|
||||
filerProvider := func() string { return "localhost:8888" }
|
||||
defaultSigningKeyProvider := func() string { return "default-secure-signing-key" }
|
||||
|
||||
// Load
|
||||
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, manager)
|
||||
}
|
||||
|
||||
func TestLoadIAMManagerFromConfig_PartialDefaults(t *testing.T) {
|
||||
// Test that partial configs (e.g. providing SigningKey but not Duration) work
|
||||
tmpDir := t.TempDir()
|
||||
configPath := filepath.Join(tmpDir, "iam_config_partial.json")
|
||||
|
||||
// Signing key provided in JSON, others missing
|
||||
configContent := `{
|
||||
"sts": {
|
||||
"signingKey": "anNvbi1wcm92aWRlZC1rZXktMTIzNDU="
|
||||
},
|
||||
"policy": {
|
||||
"storeType": "memory",
|
||||
"defaultEffect": "Allow"
|
||||
}
|
||||
}`
|
||||
|
||||
err := os.WriteFile(configPath, []byte(configContent), 0644)
|
||||
assert.NoError(t, err)
|
||||
|
||||
filerProvider := func() string { return "localhost:8888" }
|
||||
// Default signing key provided but should be IGNORED because JSON has one
|
||||
defaultSigningKeyProvider := func() string { return "server-default-key-should-be-ignored" }
|
||||
|
||||
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, manager)
|
||||
}
|
||||
|
||||
func TestLoadIAMManagerFromConfig_ExplicitEmptyKey(t *testing.T) {
|
||||
// Test that if JSON has empty signing key string, it still falls back
|
||||
tmpDir := t.TempDir()
|
||||
configPath := filepath.Join(tmpDir, "iam_config_empty_key.json")
|
||||
|
||||
// Signing key explicitly empty
|
||||
configContent := `{
|
||||
"sts": {
|
||||
"signingKey": ""
|
||||
},
|
||||
"policy": {
|
||||
"storeType": "memory",
|
||||
"defaultEffect": "Allow"
|
||||
}
|
||||
}`
|
||||
|
||||
err := os.WriteFile(configPath, []byte(configContent), 0644)
|
||||
assert.NoError(t, err)
|
||||
|
||||
filerProvider := func() string { return "localhost:8888" }
|
||||
defaultSigningKeyProvider := func() string { return "fallback-key-should-be-used" }
|
||||
|
||||
manager, err := loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, manager)
|
||||
}
|
||||
|
||||
func TestLoadIAMManagerFromConfig_MissingKeyError(t *testing.T) {
|
||||
// Test that if BOTH keys are empty, it fails with a clear error
|
||||
tmpDir := t.TempDir()
|
||||
configPath := filepath.Join(tmpDir, "iam_config_all_empty.json")
|
||||
|
||||
// Signing key explicitly empty in JSON
|
||||
configContent := `{
|
||||
"sts": {
|
||||
"signingKey": ""
|
||||
},
|
||||
"policy": {
|
||||
"storeType": "memory",
|
||||
"defaultEffect": "Allow"
|
||||
}
|
||||
}`
|
||||
|
||||
err := os.WriteFile(configPath, []byte(configContent), 0644)
|
||||
assert.NoError(t, err)
|
||||
|
||||
filerProvider := func() string { return "localhost:8888" }
|
||||
defaultSigningKeyProvider := func() string { return "" } // Empty default too
|
||||
|
||||
// Ensure no SSE-S3 key interferes (global state in tests is tricky, but let's assume clean state or no mock)
|
||||
// Ideally we would mock GetSSES3KeyManager().GetMasterKey() but it's a global singleton.
|
||||
// For this unit test, if the global key manager has no key, it should fail.
|
||||
|
||||
_, err = loadIAMManagerFromConfig(configPath, filerProvider, defaultSigningKeyProvider)
|
||||
|
||||
// Should return a clear error
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "no signing key found for STS service")
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLoadIAMManagerFromConfig_OptionalConfig(t *testing.T) {
|
||||
// Mock dependencies
|
||||
filerAddressProvider := func() string { return "localhost:8888" }
|
||||
getFilerSigningKey := func() string { return "test-signing-key" }
|
||||
|
||||
// Test Case 1: Empty config path should load defaults
|
||||
iamManager, err := loadIAMManagerFromConfig("", filerAddressProvider, getFilerSigningKey)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, iamManager)
|
||||
|
||||
// Verify STS Service is initialized with defaults
|
||||
stsService := iamManager.GetSTSService()
|
||||
assert.NotNil(t, stsService)
|
||||
|
||||
// Verify defaults are applied
|
||||
// Since we can't easily access the internal config of stsService,
|
||||
// we rely on the fact that initialization succeeded without error.
|
||||
// We can also verify that the policy engine uses memory store by default.
|
||||
|
||||
// Verify Policy Engine is initialized with defaults (Memory store, Deny effect)
|
||||
// Again, internal state might be hard to access directly, but successful init implies defaults worked.
|
||||
}
|
||||
|
||||
func TestLoadIAMManagerFromConfig_EmptyConfigWithFallbackKey(t *testing.T) {
|
||||
// Mock dependencies where getFilerSigningKey returns empty, forcing fallback logic
|
||||
// Initialize IAM with empty config (should trigger defaults)
|
||||
// We pass empty string for config file path
|
||||
option := &S3ApiServerOption{
|
||||
Config: "",
|
||||
IamConfig: "",
|
||||
EnableIam: true,
|
||||
}
|
||||
iamManager := NewIdentityAccessManagementWithStore(option, nil, "memory")
|
||||
|
||||
// Verify identityAnonymous is initialized
|
||||
// This confirms the fix for anonymous access in zero-config mode
|
||||
anonIdentity, found := iamManager.LookupAnonymous()
|
||||
assert.True(t, found, "Anonymous identity should be found by default")
|
||||
assert.NotNil(t, anonIdentity, "Anonymous identity should not be nil")
|
||||
assert.Equal(t, "anonymous", anonIdentity.Name)
|
||||
}
|
||||
@@ -18,7 +18,6 @@ type FilerClient interface {
|
||||
|
||||
type S3Authenticator interface {
|
||||
AuthenticateRequest(r *http.Request) (string, interface{}, s3err.ErrorCode)
|
||||
DefaultAllow() bool
|
||||
}
|
||||
|
||||
// Server implements the Iceberg REST Catalog API.
|
||||
@@ -129,25 +128,20 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc {
|
||||
|
||||
identityName, identity, errCode := s.authenticator.AuthenticateRequest(r)
|
||||
if errCode != s3err.ErrNone {
|
||||
// If authentication failed but DefaultAllow is enabled, proceed without identity
|
||||
if s.authenticator.DefaultAllow() {
|
||||
glog.V(2).Infof("Iceberg: AuthenticateRequest failed (%v), but DefaultAllow is true, proceeding", errCode)
|
||||
} else {
|
||||
apiErr := s3err.GetAPIError(errCode)
|
||||
errorType := "RESTException"
|
||||
switch apiErr.HTTPStatusCode {
|
||||
case http.StatusForbidden:
|
||||
errorType = "ForbiddenException"
|
||||
case http.StatusUnauthorized:
|
||||
errorType = "NotAuthorizedException"
|
||||
case http.StatusBadRequest:
|
||||
errorType = "BadRequestException"
|
||||
case http.StatusInternalServerError:
|
||||
errorType = "InternalServerError"
|
||||
}
|
||||
writeError(w, apiErr.HTTPStatusCode, errorType, apiErr.Description)
|
||||
return
|
||||
apiErr := s3err.GetAPIError(errCode)
|
||||
errorType := "RESTException"
|
||||
switch apiErr.HTTPStatusCode {
|
||||
case http.StatusForbidden:
|
||||
errorType = "ForbiddenException"
|
||||
case http.StatusUnauthorized:
|
||||
errorType = "NotAuthorizedException"
|
||||
case http.StatusBadRequest:
|
||||
errorType = "BadRequestException"
|
||||
case http.StatusInternalServerError:
|
||||
errorType = "InternalServerError"
|
||||
}
|
||||
writeError(w, apiErr.HTTPStatusCode, errorType, apiErr.Description)
|
||||
return
|
||||
}
|
||||
|
||||
if identityName != "" || identity != nil {
|
||||
|
||||
@@ -44,7 +44,6 @@ type IAMIntegration interface {
|
||||
AuthorizeAction(ctx context.Context, identity *IAMIdentity, action Action, bucket string, objectKey string, r *http.Request) s3err.ErrorCode
|
||||
ValidateSessionToken(ctx context.Context, token string) (*sts.SessionInfo, error)
|
||||
ValidateTrustPolicyForPrincipal(ctx context.Context, roleArn, principalArn string) error
|
||||
DefaultAllow() bool
|
||||
}
|
||||
|
||||
// S3IAMIntegration provides IAM integration for S3 API
|
||||
@@ -311,14 +310,6 @@ func (s3iam *S3IAMIntegration) ValidateTrustPolicyForPrincipal(ctx context.Conte
|
||||
return s3iam.iamManager.ValidateTrustPolicyForPrincipal(ctx, roleArn, principalArn)
|
||||
}
|
||||
|
||||
// DefaultAllow returns whether access is allowed by default when no policy is found
|
||||
func (s3iam *S3IAMIntegration) DefaultAllow() bool {
|
||||
if s3iam.iamManager == nil {
|
||||
return true // Default to true if IAM is not enabled
|
||||
}
|
||||
return s3iam.iamManager.DefaultAllow()
|
||||
}
|
||||
|
||||
// IAMIdentity represents an authenticated identity with session information
|
||||
type IAMIdentity struct {
|
||||
Name string
|
||||
|
||||
+2
-50
@@ -5,7 +5,6 @@ import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
@@ -20,13 +19,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||
"golang.org/x/crypto/hkdf"
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
// SSE-S3 uses AES-256 encryption with server-managed keys
|
||||
@@ -457,27 +452,6 @@ func (km *SSES3KeyManager) GetKey(keyID string) (*SSES3Key, bool) {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// GetMasterKey returns a derived key from the master KEK for STS signing
|
||||
// This uses HKDF to isolate the STS security domain from the SSE-S3 domain
|
||||
func (km *SSES3KeyManager) GetMasterKey() []byte {
|
||||
km.mu.RLock()
|
||||
defer km.mu.RUnlock()
|
||||
|
||||
if len(km.superKey) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Derive a separate key for STS to isolate security domains
|
||||
// We use the KEK as the secret, and "seaweedfs-sts-signing-key" as the info
|
||||
hkdfReader := hkdf.New(sha256.New, km.superKey, nil, []byte("seaweedfs-sts-signing-key"))
|
||||
derived := make([]byte, 32) // 256-bit derived key
|
||||
if _, err := io.ReadFull(hkdfReader, derived); err != nil {
|
||||
glog.Errorf("Failed to derive STS key: %v", err)
|
||||
return nil
|
||||
}
|
||||
return derived
|
||||
}
|
||||
|
||||
// Global SSE-S3 key manager instance
|
||||
var globalSSES3KeyManager = NewSSES3KeyManager()
|
||||
|
||||
@@ -486,31 +460,9 @@ func GetSSES3KeyManager() *SSES3KeyManager {
|
||||
return globalSSES3KeyManager
|
||||
}
|
||||
|
||||
// KeyManagerFilerClient wraps wdclient.FilerClient to satisfy filer_pb.FilerClient interface
|
||||
type KeyManagerFilerClient struct {
|
||||
*wdclient.FilerClient
|
||||
grpcDialOption grpc.DialOption
|
||||
}
|
||||
|
||||
func (k *KeyManagerFilerClient) AdjustedUrl(location *filer_pb.Location) string {
|
||||
return location.Url
|
||||
}
|
||||
|
||||
func (k *KeyManagerFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||
filerAddress := k.GetCurrentFiler()
|
||||
if filerAddress == "" {
|
||||
return fmt.Errorf("no filer available")
|
||||
}
|
||||
return pb.WithGrpcFilerClient(streamingMode, 0, filerAddress, k.grpcDialOption, fn)
|
||||
}
|
||||
|
||||
// InitializeGlobalSSES3KeyManager initializes the global key manager with filer access
|
||||
func InitializeGlobalSSES3KeyManager(filerClient *wdclient.FilerClient, grpcDialOption grpc.DialOption) error {
|
||||
wrapper := &KeyManagerFilerClient{
|
||||
FilerClient: filerClient,
|
||||
grpcDialOption: grpcDialOption,
|
||||
}
|
||||
return globalSSES3KeyManager.InitializeWithFiler(wrapper)
|
||||
func InitializeGlobalSSES3KeyManager(s3ApiServer *S3ApiServer) error {
|
||||
return globalSSES3KeyManager.InitializeWithFiler(s3ApiServer)
|
||||
}
|
||||
|
||||
// ProcessSSES3Request processes an SSE-S3 request and returns encryption metadata
|
||||
|
||||
@@ -19,6 +19,21 @@ const (
|
||||
deleteMultipleObjectsLimit = 1000
|
||||
)
|
||||
|
||||
// objectLockVersionToCheckForDelete resolves which version should be validated for Object Lock protections.
|
||||
// For versioned delete without explicit versionId, this targets the latest version (empty versionId for enabled,
|
||||
// "null" for suspended) because DeleteObject affects that version's visibility.
|
||||
// This always returns a version ID to check; object lock protections are fail-closed.
|
||||
func objectLockVersionToCheckForDelete(versioningState, requestedVersionID string) string {
|
||||
if requestedVersionID != "" {
|
||||
return requestedVersionID
|
||||
}
|
||||
|
||||
if versioningState == s3_constants.VersioningSuspended {
|
||||
return "null"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
bucket, object := s3_constants.GetBucketAndObject(r)
|
||||
@@ -52,18 +67,18 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
||||
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
|
||||
}
|
||||
|
||||
lockCheckVersionID := objectLockVersionToCheckForDelete(versioningState, versionId)
|
||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
|
||||
if err := s3a.enforceObjectLockProtections(r, bucket, object, lockCheckVersionID, governanceBypassAllowed); err != nil {
|
||||
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object, lockCheckVersionID, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
|
||||
if versioningConfigured {
|
||||
// Handle versioned delete based on specific versioning state
|
||||
if versionId != "" {
|
||||
// Delete specific version (same for both enabled and suspended)
|
||||
// Check object lock permissions before deleting specific version
|
||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
|
||||
if err := s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed); err != nil {
|
||||
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
|
||||
// Delete specific version
|
||||
err := s3a.deleteSpecificObjectVersion(bucket, object, versionId)
|
||||
if err != nil {
|
||||
@@ -77,9 +92,7 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
||||
} else {
|
||||
// Delete without version ID - behavior depends on versioning state
|
||||
if versioningEnabled {
|
||||
// Enabled versioning: Create delete marker (logical delete)
|
||||
// AWS S3 behavior: Delete marker creation is NOT blocked by object retention
|
||||
// because it's a logical delete that doesn't actually remove the retained version
|
||||
// Enabled versioning: create delete marker (logical delete)
|
||||
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to create delete marker: %v", err)
|
||||
@@ -94,14 +107,6 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
||||
// Suspended versioning: Actually delete the "null" version object
|
||||
glog.V(2).Infof("DeleteObjectHandler: deleting null version for suspended versioning %s/%s", bucket, object)
|
||||
|
||||
// Check object lock permissions before deleting "null" version
|
||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
|
||||
if err := s3a.enforceObjectLockProtections(r, bucket, object, "null", governanceBypassAllowed); err != nil {
|
||||
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
|
||||
// Delete the "null" version (the regular file)
|
||||
err := s3a.deleteSpecificObjectVersion(bucket, object, "null")
|
||||
if err != nil {
|
||||
@@ -116,14 +121,6 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
} else {
|
||||
// Handle regular delete (non-versioned)
|
||||
// Check object lock permissions before deleting object
|
||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
|
||||
if err := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); err != nil {
|
||||
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
|
||||
// Normalize trailing-slash object keys (e.g. "path/") to the
|
||||
// underlying directory entry path so DeleteEntry gets a valid name.
|
||||
target := util.NewFullPath(s3a.bucketDir(bucket), object)
|
||||
@@ -249,20 +246,19 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
||||
continue
|
||||
}
|
||||
|
||||
// Check object lock permissions before deletion (only for versioned buckets)
|
||||
if versioningConfigured {
|
||||
// Validate governance bypass for this specific object
|
||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key)
|
||||
if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, object.VersionId, governanceBypassAllowed); err != nil {
|
||||
glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object.Key, object.VersionId, err)
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: s3err.GetAPIError(s3err.ErrAccessDenied).Code,
|
||||
Message: s3err.GetAPIError(s3err.ErrAccessDenied).Description,
|
||||
Key: object.Key,
|
||||
VersionId: object.VersionId,
|
||||
})
|
||||
continue
|
||||
}
|
||||
// Check object lock permissions before deletion (applies to all buckets: versioned or non-versioned)
|
||||
lockCheckVersionID := objectLockVersionToCheckForDelete(versioningState, object.VersionId)
|
||||
// Validate governance bypass for this specific object
|
||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key)
|
||||
if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, lockCheckVersionID, governanceBypassAllowed); err != nil {
|
||||
glog.V(2).Infof("DeleteMultipleObjectsHandler: object lock check failed for %s/%s (version: %s): %v", bucket, object.Key, lockCheckVersionID, err)
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: s3err.GetAPIError(s3err.ErrAccessDenied).Code,
|
||||
Message: s3err.GetAPIError(s3err.ErrAccessDenied).Description,
|
||||
Key: object.Key,
|
||||
VersionId: object.VersionId,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
var deleteVersionId string
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestObjectLockVersionToCheckForDelete(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
versioningState string
|
||||
requestedVersionID string
|
||||
expectedVersionID string
|
||||
}{
|
||||
{
|
||||
name: "enabled versioning without version id checks latest version",
|
||||
versioningState: s3_constants.VersioningEnabled,
|
||||
requestedVersionID: "",
|
||||
expectedVersionID: "",
|
||||
},
|
||||
{
|
||||
name: "suspended versioning without version id checks null version",
|
||||
versioningState: s3_constants.VersioningSuspended,
|
||||
requestedVersionID: "",
|
||||
expectedVersionID: "null",
|
||||
},
|
||||
{
|
||||
name: "specific version id is always checked",
|
||||
versioningState: s3_constants.VersioningEnabled,
|
||||
requestedVersionID: "3LgYQ7f7VxQ3",
|
||||
expectedVersionID: "3LgYQ7f7VxQ3",
|
||||
},
|
||||
{
|
||||
name: "non-versioned buckets still check current object",
|
||||
versioningState: "",
|
||||
requestedVersionID: "",
|
||||
expectedVersionID: "",
|
||||
},
|
||||
{
|
||||
name: "unknown versioning state defaults to empty version",
|
||||
versioningState: "UnexpectedState",
|
||||
requestedVersionID: "",
|
||||
expectedVersionID: "",
|
||||
},
|
||||
{
|
||||
name: "suspended versioning with specific version id checks that version",
|
||||
versioningState: s3_constants.VersioningSuspended,
|
||||
requestedVersionID: "abc123",
|
||||
expectedVersionID: "abc123",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
versionID := objectLockVersionToCheckForDelete(tc.versioningState, tc.requestedVersionID)
|
||||
assert.Equal(t, tc.expectedVersionID, versionID)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
func TestGetRequestDataReader_ChunkedEncodingWithoutIAM(t *testing.T) {
|
||||
// Create an S3ApiServer with IAM disabled
|
||||
s3a := &S3ApiServer{
|
||||
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, string(credential.StoreTypeMemory)),
|
||||
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
|
||||
}
|
||||
// Ensure IAM is disabled for this test
|
||||
s3a.iam.isAuthEnabled = false
|
||||
@@ -87,7 +87,7 @@ func TestGetRequestDataReader_ChunkedEncodingWithoutIAM(t *testing.T) {
|
||||
func TestGetRequestDataReader_AuthTypeDetection(t *testing.T) {
|
||||
// Create an S3ApiServer with IAM disabled
|
||||
s3a := &S3ApiServer{
|
||||
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, string(credential.StoreTypeMemory)),
|
||||
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
|
||||
}
|
||||
s3a.iam.isAuthEnabled = false
|
||||
|
||||
@@ -122,7 +122,7 @@ func TestGetRequestDataReader_AuthTypeDetection(t *testing.T) {
|
||||
func TestGetRequestDataReader_IAMEnabled(t *testing.T) {
|
||||
// Create an S3ApiServer with IAM enabled
|
||||
s3a := &S3ApiServer{
|
||||
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, nil, string(credential.StoreTypeMemory)),
|
||||
iam: NewIdentityAccessManagementWithStore(&S3ApiServerOption{}, string(credential.StoreTypeMemory)),
|
||||
}
|
||||
s3a.iam.isAuthEnabled = true
|
||||
|
||||
|
||||
+40
-94
@@ -110,8 +110,7 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
|
||||
option.AllowedOrigins = domains
|
||||
}
|
||||
|
||||
// Initialize basic/legacy IAM - filerClient not available yet, passed as nil
|
||||
iam := NewIdentityAccessManagementWithStore(option, nil, explicitStore)
|
||||
iam := NewIdentityAccessManagementWithStore(option, explicitStore)
|
||||
|
||||
// Initialize bucket policy engine first
|
||||
policyEngine := NewBucketPolicyEngine()
|
||||
@@ -147,24 +146,16 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
|
||||
glog.V(1).Infof("S3 API initialized FilerClient with %d filer(s) (no discovery)", len(option.Filers))
|
||||
}
|
||||
|
||||
// Initialize Global SSE-S3 Key Manager early so it's available for IAM fallback
|
||||
// This ensures we can access the KEK for STS signing key if needed
|
||||
if err := InitializeGlobalSSES3KeyManager(filerClient, option.GrpcDialOption); err != nil {
|
||||
glog.Errorf("Failed to initialize SSE-S3 Key Manager: %v", err)
|
||||
// We continue, as this might be a transient failure or non-critical for some setups,
|
||||
// but IAM fallback to KEK will fail if this didn't succeed.
|
||||
}
|
||||
|
||||
// Update credential store to use FilerClient's current filer for HA
|
||||
iam.SetFilerClient(filerClient)
|
||||
|
||||
// Keep attempting to load configuration from filer now that we have a client
|
||||
// The initial load in NewIdentityAccessManagementWithStore might have failed if client was nil
|
||||
go func() {
|
||||
if err := iam.loadS3ApiConfigurationFromFiler(option); err != nil {
|
||||
glog.Warningf("Failed to load IAM config from filer after client update: %v", err)
|
||||
if store := iam.credentialManager.GetStore(); store != nil {
|
||||
if filerFuncSetter, ok := store.(interface {
|
||||
SetFilerAddressFunc(func() pb.ServerAddress, grpc.DialOption)
|
||||
}); ok {
|
||||
// Use FilerClient's GetCurrentFiler for true HA
|
||||
filerFuncSetter.SetFilerAddressFunc(filerClient.GetCurrentFiler, option.GrpcDialOption)
|
||||
glog.V(1).Infof("Updated credential store to use FilerClient's current active filer (HA-aware)")
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
s3ApiServer = &S3ApiServer{
|
||||
option: option,
|
||||
@@ -187,25 +178,19 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
|
||||
// This avoids circular dependency by not passing the entire S3ApiServer
|
||||
iam.policyEngine = policyEngine
|
||||
|
||||
// Initialize advanced IAM system if config is provided or explicitly enabled
|
||||
if option.IamConfig != "" || option.EnableIam {
|
||||
configSource := "defaults"
|
||||
if option.IamConfig != "" {
|
||||
configSource = option.IamConfig
|
||||
}
|
||||
glog.V(1).Infof("Loading advanced IAM configuration from: %s", configSource)
|
||||
// Initialize advanced IAM system if config is provided
|
||||
if option.IamConfig != "" {
|
||||
glog.V(1).Infof("Loading advanced IAM configuration from: %s", option.IamConfig)
|
||||
|
||||
// Use FilerClient's GetCurrentFiler for HA-aware filer selection
|
||||
iamManager, err := loadIAMManagerFromConfig(option.IamConfig, func() string {
|
||||
return string(filerClient.GetCurrentFiler())
|
||||
}, func() string {
|
||||
return signingKey
|
||||
})
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to load IAM configuration: %v", err)
|
||||
} else {
|
||||
if s3ApiServer.iam.credentialManager != nil {
|
||||
iamManager.SetUserStore(s3ApiServer.iam.credentialManager)
|
||||
if iam.credentialManager != nil {
|
||||
iamManager.SetUserStore(iam.credentialManager)
|
||||
}
|
||||
glog.V(1).Infof("IAM Manager loaded, creating integration")
|
||||
// Create S3 IAM integration with the loaded IAM manager
|
||||
@@ -248,10 +233,6 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
|
||||
})
|
||||
}
|
||||
s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer)
|
||||
|
||||
// Update IAM with the final filer client (already handled by SetFilerClient above,
|
||||
// but this reinforces it if we ever change the flow)
|
||||
s3ApiServer.iam.SetFilerClient(s3ApiServer.filerClient)
|
||||
if option.LocalFilerSocket == "" {
|
||||
if s3ApiServer.client, err = util_http.NewGlobalHttpClient(); err != nil {
|
||||
return nil, err
|
||||
@@ -268,6 +249,11 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl
|
||||
|
||||
s3ApiServer.registerRouter(router)
|
||||
|
||||
// Initialize the global SSE-S3 key manager with filer access
|
||||
if err := InitializeGlobalSSES3KeyManager(s3ApiServer); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize SSE-S3 key manager: %w", err)
|
||||
}
|
||||
|
||||
go s3ApiServer.subscribeMetaEvents("s3", startTsNs, filer.DirectoryEtcRoot, []string{
|
||||
option.BucketsPath,
|
||||
filer.IamConfigDirectory,
|
||||
@@ -844,7 +830,14 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
|
||||
}
|
||||
|
||||
// loadIAMManagerFromConfig loads the advanced IAM manager from configuration file
|
||||
func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() string, getFilerSigningKey func() string) (*integration.IAMManager, error) {
|
||||
func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() string) (*integration.IAMManager, error) {
|
||||
// Read configuration file
|
||||
configData, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read config file: %w", err)
|
||||
}
|
||||
|
||||
// Parse configuration structure
|
||||
var configRoot struct {
|
||||
STS *sts.STSConfig `json:"sts"`
|
||||
Policy *policy.PolicyEngineConfig `json:"policy"`
|
||||
@@ -856,43 +849,24 @@ func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() str
|
||||
} `json:"policies"`
|
||||
}
|
||||
|
||||
if configPath != "" {
|
||||
// Read configuration file
|
||||
configData, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read config file: %w", err)
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(configData, &configRoot); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
} else {
|
||||
glog.V(1).Infof("No IAM config file provided; using defaults")
|
||||
// Initialize with empty config which will trigger defaults below
|
||||
}
|
||||
|
||||
// Ensure STS config exists so we can apply defaults later
|
||||
if configRoot.STS == nil {
|
||||
configRoot.STS = &sts.STSConfig{}
|
||||
if err := json.Unmarshal(configData, &configRoot); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Ensure a valid policy engine config exists
|
||||
if configRoot.Policy == nil {
|
||||
configRoot.Policy = &policy.PolicyEngineConfig{}
|
||||
}
|
||||
if configRoot.Policy.StoreType == "" {
|
||||
configRoot.Policy.StoreType = sts.StoreTypeMemory
|
||||
}
|
||||
if configRoot.Policy.DefaultEffect == "" {
|
||||
// Default to Allow (open) with in-memory store so that
|
||||
// users can start using STS without locking themselves out immediately.
|
||||
// For other stores (e.g. filer), default to Deny (closed) for security.
|
||||
if configRoot.Policy.StoreType == sts.StoreTypeMemory {
|
||||
configRoot.Policy.DefaultEffect = sts.EffectAllow
|
||||
} else {
|
||||
configRoot.Policy.DefaultEffect = sts.EffectDeny
|
||||
// Provide a secure default if not specified in the config file
|
||||
// Default to Deny with in-memory store so that JSON-defined policies work without filer
|
||||
glog.V(1).Infof("No policy engine config provided; using defaults (DefaultEffect=%s, StoreType=%s)", sts.EffectDeny, sts.StoreTypeMemory)
|
||||
configRoot.Policy = &policy.PolicyEngineConfig{
|
||||
DefaultEffect: sts.EffectDeny,
|
||||
StoreType: sts.StoreTypeMemory,
|
||||
}
|
||||
glog.V(1).Infof("Using policy defaults: DefaultEffect=%s, StoreType=%s", configRoot.Policy.DefaultEffect, configRoot.Policy.StoreType)
|
||||
} else if configRoot.Policy.StoreType == "" {
|
||||
// If policy config exists but storeType is not specified, use memory store
|
||||
// This ensures JSON-defined policies are stored in memory and work correctly
|
||||
configRoot.Policy.StoreType = sts.StoreTypeMemory
|
||||
glog.V(1).Infof("Policy storeType not specified; using memory store for JSON config-based setup")
|
||||
}
|
||||
|
||||
// Create IAM configuration
|
||||
@@ -904,26 +878,6 @@ func loadIAMManagerFromConfig(configPath string, filerAddressProvider func() str
|
||||
},
|
||||
}
|
||||
|
||||
// Apply default signing key if not present in config
|
||||
if iamConfig.STS != nil && len(iamConfig.STS.SigningKey) == 0 {
|
||||
// 1. Try server-configured signing key (security.toml / CLI)
|
||||
if key := getFilerSigningKey(); key != "" {
|
||||
iamConfig.STS.SigningKey = []byte(key)
|
||||
glog.V(1).Infof("Using default filer signing key for STS service")
|
||||
} else {
|
||||
// 2. Try cluster-wide SSE-S3 Master Key (KEK) from Filer
|
||||
// This ensures zero-config consistency across the cluster
|
||||
if kek := GetSSES3KeyManager().GetMasterKey(); len(kek) > 0 {
|
||||
iamConfig.STS.SigningKey = kek
|
||||
glog.V(1).Infof("Using SSE-S3 Master Key (KEK) for STS service")
|
||||
} else {
|
||||
// 3. Fail if no signing key is available
|
||||
// This ensures consistency across multiple S3 servers and secure operation
|
||||
return nil, fmt.Errorf("no signing key found for STS service; please provide 'signingKey' in IAM config, configure 'jwt.filer_signing.key' in security.toml, or ensure SSE-S3 is initialized")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize IAM manager
|
||||
iamManager := integration.NewIAMManager()
|
||||
if err := iamManager.Initialize(iamConfig, filerAddressProvider); err != nil {
|
||||
@@ -1006,11 +960,3 @@ func (s3a *S3ApiServer) AuthenticateRequest(r *http.Request) (string, interface{
|
||||
}
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
// DefaultAllow returns whether access is allowed by default when no policy is found
|
||||
func (s3a *S3ApiServer) DefaultAllow() bool {
|
||||
if s3a.iam == nil || s3a.iam.iamIntegration == nil {
|
||||
return false
|
||||
}
|
||||
return s3a.iam.iamIntegration.DefaultAllow()
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
// setupRoutingTestServer creates a minimal S3ApiServer for routing tests
|
||||
func setupRoutingTestServer(t *testing.T) *S3ApiServer {
|
||||
opt := &S3ApiServerOption{EnableIam: true}
|
||||
iam := NewIdentityAccessManagementWithStore(opt, nil, "memory")
|
||||
iam := NewIdentityAccessManagementWithStore(opt, "memory")
|
||||
iam.isAuthEnabled = true
|
||||
|
||||
if iam.credentialManager == nil {
|
||||
|
||||
@@ -43,11 +43,6 @@ func (st *S3TablesApiServer) SetAccountID(accountID string) {
|
||||
st.handler.SetAccountID(accountID)
|
||||
}
|
||||
|
||||
// SetDefaultAllow sets whether to allow access by default
|
||||
func (st *S3TablesApiServer) SetDefaultAllow(allow bool) {
|
||||
st.handler.SetDefaultAllow(allow)
|
||||
}
|
||||
|
||||
// S3TablesHandler handles S3 Tables API requests
|
||||
func (st *S3TablesApiServer) S3TablesHandler(w http.ResponseWriter, r *http.Request) {
|
||||
st.handler.HandleRequest(w, r, st)
|
||||
@@ -62,12 +57,6 @@ func (st *S3TablesApiServer) WithFilerClient(streamingMode bool, fn func(filer_p
|
||||
func (s3a *S3ApiServer) registerS3TablesRoutes(router *mux.Router) {
|
||||
// Create S3 Tables handler
|
||||
s3TablesApi := NewS3TablesApiServer(s3a)
|
||||
if s3a.iam != nil && s3a.iam.iamIntegration != nil {
|
||||
s3TablesApi.SetDefaultAllow(s3a.iam.iamIntegration.DefaultAllow())
|
||||
} else {
|
||||
// If IAM is not configured, allow all access by default
|
||||
s3TablesApi.SetDefaultAllow(true)
|
||||
}
|
||||
|
||||
// Regex for S3 Tables Bucket ARN
|
||||
const tableBucketARNRegex = "arn:aws:s3tables:[^/:]*:[^/:]*:bucket/[^/]+"
|
||||
@@ -629,15 +618,9 @@ func (s3a *S3ApiServer) authenticateS3Tables(f http.HandlerFunc) http.HandlerFun
|
||||
// Use AuthSignatureOnly to authenticate the request without authorizing specific actions
|
||||
identity, errCode := s3a.iam.AuthSignatureOnly(r)
|
||||
if errCode != s3err.ErrNone {
|
||||
// If IAM is enabled but DefaultAllow is true, we can proceed even if unauthenticated
|
||||
// authorization checks in handlers will then use DefaultAllow logic.
|
||||
if s3a.iam.iamIntegration != nil && s3a.iam.iamIntegration.DefaultAllow() {
|
||||
glog.V(2).Infof("S3Tables: AuthSignatureOnly failed (%v), but DefaultAllow is true, proceeding", errCode)
|
||||
} else {
|
||||
glog.Errorf("S3Tables: AuthSignatureOnly failed: %v", errCode)
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
glog.Errorf("S3Tables: AuthSignatureOnly failed: %v", errCode)
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Store the authenticated identity in request context
|
||||
|
||||
@@ -44,17 +44,15 @@ const (
|
||||
|
||||
// S3TablesHandler handles S3 Tables API requests
|
||||
type S3TablesHandler struct {
|
||||
region string
|
||||
accountID string
|
||||
defaultAllow bool // Whether to allow access by default (for zero-config IAM)
|
||||
region string
|
||||
accountID string
|
||||
}
|
||||
|
||||
// NewS3TablesHandler creates a new S3 Tables handler
|
||||
func NewS3TablesHandler() *S3TablesHandler {
|
||||
return &S3TablesHandler{
|
||||
region: DefaultRegion,
|
||||
accountID: DefaultAccountID,
|
||||
defaultAllow: false,
|
||||
region: DefaultRegion,
|
||||
accountID: DefaultAccountID,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,11 +70,6 @@ func (h *S3TablesHandler) SetAccountID(accountID string) {
|
||||
}
|
||||
}
|
||||
|
||||
// SetDefaultAllow sets whether to allow access by default
|
||||
func (h *S3TablesHandler) SetDefaultAllow(allow bool) {
|
||||
h.defaultAllow = allow
|
||||
}
|
||||
|
||||
// FilerClient interface for filer operations
|
||||
type FilerClient interface {
|
||||
WithFilerClient(streamingMode bool, fn func(client filer_pb.SeaweedFilerClient) error) error
|
||||
|
||||
@@ -16,9 +16,7 @@ import (
|
||||
func (h *S3TablesHandler) handleCreateTableBucket(w http.ResponseWriter, r *http.Request, filerClient FilerClient) error {
|
||||
// Check permission
|
||||
principal := h.getAccountID(r)
|
||||
if !CheckPermissionWithContext("CreateTableBucket", principal, principal, "", "", &PolicyContext{
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
if !CanCreateTableBucket(principal, principal, "") {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create table buckets")
|
||||
return NewAuthError("CreateTableBucket", principal, "not authorized to create table buckets")
|
||||
}
|
||||
|
||||
@@ -72,7 +72,6 @@ func (h *S3TablesHandler) handleGetTableBucket(w http.ResponseWriter, r *http.Re
|
||||
if !CheckPermissionWithContext("GetTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket details")
|
||||
return ErrAccessDenied
|
||||
@@ -102,7 +101,6 @@ func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.
|
||||
identityActions := getIdentityActions(r)
|
||||
if !CheckPermissionWithContext("ListTableBuckets", principal, accountID, "", "", &PolicyContext{
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to list table buckets")
|
||||
return NewAuthError("ListTableBuckets", principal, "not authorized to list table buckets")
|
||||
@@ -200,7 +198,6 @@ func (h *S3TablesHandler) handleListTableBuckets(w http.ResponseWriter, r *http.
|
||||
if !CheckPermissionWithContext("GetTableBucket", accountID, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: entry.Entry.Name,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
@@ -303,7 +300,6 @@ func (h *S3TablesHandler) handleDeleteTableBucket(w http.ResponseWriter, r *http
|
||||
if !CheckPermissionWithContext("DeleteTableBucket", principal, metadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
return NewAuthError("DeleteTableBucket", principal, fmt.Sprintf("not authorized to delete bucket %s", bucketName))
|
||||
}
|
||||
|
||||
@@ -118,7 +118,6 @@ func (h *S3TablesHandler) handleCreateNamespace(w http.ResponseWriter, r *http.R
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
glog.Infof("S3Tables: Permission denied for CreateNamespace - principal=%s, owner=%s", principal, bucketMetadata.OwnerAccountID)
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to create namespace in this bucket")
|
||||
@@ -259,7 +258,6 @@ func (h *S3TablesHandler) handleGetNamespace(w http.ResponseWriter, r *http.Requ
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
|
||||
return ErrAccessDenied
|
||||
@@ -346,7 +344,6 @@ func (h *S3TablesHandler) handleListNamespaces(w http.ResponseWriter, r *http.Re
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchBucket, fmt.Sprintf("table bucket %s not found", bucketName))
|
||||
return ErrAccessDenied
|
||||
@@ -531,7 +528,6 @@ func (h *S3TablesHandler) handleDeleteNamespace(w http.ResponseWriter, r *http.R
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusNotFound, ErrCodeNoSuchNamespace, "namespace not found")
|
||||
return ErrAccessDenied
|
||||
|
||||
@@ -94,7 +94,6 @@ func (h *S3TablesHandler) handlePutTableBucketPolicy(w http.ResponseWriter, r *h
|
||||
if !CheckPermissionWithContext("PutTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, "", bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table bucket policy")
|
||||
return NewAuthError("PutTableBucketPolicy", principal, "not authorized to put table bucket policy")
|
||||
@@ -172,7 +171,6 @@ func (h *S3TablesHandler) handleGetTableBucketPolicy(w http.ResponseWriter, r *h
|
||||
if !CheckPermissionWithContext("GetTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, string(policy), bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table bucket policy")
|
||||
return NewAuthError("GetTableBucketPolicy", principal, "not authorized to get table bucket policy")
|
||||
@@ -248,7 +246,6 @@ func (h *S3TablesHandler) handleDeleteTableBucketPolicy(w http.ResponseWriter, r
|
||||
if !CheckPermissionWithContext("DeleteTableBucketPolicy", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table bucket policy")
|
||||
return NewAuthError("DeleteTableBucketPolicy", principal, "not authorized to delete table bucket policy")
|
||||
@@ -349,7 +346,6 @@ func (h *S3TablesHandler) handlePutTablePolicy(w http.ResponseWriter, r *http.Re
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to put table policy")
|
||||
return NewAuthError("PutTablePolicy", principal, "not authorized to put table policy")
|
||||
@@ -457,7 +453,6 @@ func (h *S3TablesHandler) handleGetTablePolicy(w http.ResponseWriter, r *http.Re
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to get table policy")
|
||||
return NewAuthError("GetTablePolicy", principal, "not authorized to get table policy")
|
||||
@@ -547,7 +542,6 @@ func (h *S3TablesHandler) handleDeleteTablePolicy(w http.ResponseWriter, r *http
|
||||
Namespace: namespaceName,
|
||||
TableName: tableName,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table policy")
|
||||
return NewAuthError("DeleteTablePolicy", principal, "not authorized to delete table policy")
|
||||
@@ -646,7 +640,6 @@ func (h *S3TablesHandler) handleTagResource(w http.ResponseWriter, r *http.Reque
|
||||
TagKeys: requestTagKeys,
|
||||
ResourceTags: existingTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
return NewAuthError("TagResource", principal, "not authorized to tag resource")
|
||||
}
|
||||
@@ -764,7 +757,6 @@ func (h *S3TablesHandler) handleListTagsForResource(w http.ResponseWriter, r *ht
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
return NewAuthError("ListTagsForResource", principal, "not authorized to list tags for resource")
|
||||
}
|
||||
@@ -872,7 +864,6 @@ func (h *S3TablesHandler) handleUntagResource(w http.ResponseWriter, r *http.Req
|
||||
TagKeys: req.TagKeys,
|
||||
ResourceTags: tags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
return NewAuthError("UntagResource", principal, "not authorized to untag resource")
|
||||
}
|
||||
|
||||
@@ -145,7 +145,6 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
|
||||
TagKeys: mapKeys(req.Tags),
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
})
|
||||
bucketAllowed := CheckPermissionWithContext("CreateTable", accountID, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
@@ -155,7 +154,6 @@ func (h *S3TablesHandler) handleCreateTable(w http.ResponseWriter, r *http.Reque
|
||||
TagKeys: mapKeys(req.Tags),
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
})
|
||||
|
||||
if !nsAllowed && !bucketAllowed {
|
||||
@@ -392,7 +390,6 @@ func (h *S3TablesHandler) handleGetTable(w http.ResponseWriter, r *http.Request,
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tableTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
})
|
||||
bucketAllowed := CheckPermissionWithContext("GetTable", accountID, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
@@ -401,7 +398,6 @@ func (h *S3TablesHandler) handleGetTable(w http.ResponseWriter, r *http.Request,
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tableTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
})
|
||||
|
||||
if !tableAllowed && !bucketAllowed {
|
||||
@@ -531,14 +527,12 @@ func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Reques
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
})
|
||||
bucketAllowed := CheckPermissionWithContext("ListTables", accountID, bucketMeta.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
Namespace: namespaceName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
})
|
||||
if !nsAllowed && !bucketAllowed {
|
||||
return ErrAccessDenied
|
||||
@@ -580,7 +574,6 @@ func (h *S3TablesHandler) handleListTables(w http.ResponseWriter, r *http.Reques
|
||||
TableBucketName: bucketName,
|
||||
TableBucketTags: bucketTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
}) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
@@ -917,7 +910,6 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tableTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
})
|
||||
bucketAllowed := CheckPermissionWithContext("DeleteTable", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
@@ -926,7 +918,6 @@ func (h *S3TablesHandler) handleDeleteTable(w http.ResponseWriter, r *http.Reque
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tableTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
})
|
||||
if !tableAllowed && !bucketAllowed {
|
||||
h.writeError(w, http.StatusForbidden, ErrCodeAccessDenied, "not authorized to delete table")
|
||||
@@ -1062,7 +1053,6 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tableTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
})
|
||||
bucketAllowed := CheckPermissionWithContext("UpdateTable", principal, bucketMetadata.OwnerAccountID, bucketPolicy, bucketARN, &PolicyContext{
|
||||
TableBucketName: bucketName,
|
||||
@@ -1071,7 +1061,6 @@ func (h *S3TablesHandler) handleUpdateTable(w http.ResponseWriter, r *http.Reque
|
||||
TableBucketTags: bucketTags,
|
||||
ResourceTags: tableTags,
|
||||
IdentityActions: identityActions,
|
||||
DefaultAllow: h.defaultAllow,
|
||||
})
|
||||
|
||||
if !tableAllowed && !bucketAllowed {
|
||||
|
||||
@@ -20,10 +20,7 @@ type Manager struct {
|
||||
|
||||
// NewManager creates a new Manager.
|
||||
func NewManager() *Manager {
|
||||
m := &Manager{handler: NewS3TablesHandler()}
|
||||
// Default to allowing access when IAM is not configured
|
||||
m.handler.SetDefaultAllow(true)
|
||||
return m
|
||||
return &Manager{handler: NewS3TablesHandler()}
|
||||
}
|
||||
|
||||
// SetRegion sets the AWS region for ARN generation.
|
||||
@@ -36,11 +33,6 @@ func (m *Manager) SetAccountID(accountID string) {
|
||||
m.handler.SetAccountID(accountID)
|
||||
}
|
||||
|
||||
// SetDefaultAllow sets whether to allow access by default.
|
||||
func (m *Manager) SetDefaultAllow(allow bool) {
|
||||
m.handler.SetDefaultAllow(allow)
|
||||
}
|
||||
|
||||
// Execute runs an S3 Tables operation and decodes the response into resp (if provided).
|
||||
func (m *Manager) Execute(ctx context.Context, filerClient FilerClient, operation string, req interface{}, resp interface{}, identity string) error {
|
||||
body, err := json.Marshal(req)
|
||||
|
||||
@@ -86,7 +86,6 @@ type PolicyContext struct {
|
||||
SSEAlgorithm string
|
||||
KMSKeyArn string
|
||||
StorageClass string
|
||||
DefaultAllow bool
|
||||
}
|
||||
|
||||
// CheckPermissionWithResource checks if a principal has permission to perform an operation on a specific resource
|
||||
@@ -118,7 +117,6 @@ func CheckPermissionWithContext(operation, principal, owner, resourcePolicy, res
|
||||
}
|
||||
|
||||
func checkPermission(operation, principal, owner, resourcePolicy, resourceARN string, ctx *PolicyContext) bool {
|
||||
|
||||
// Owner always has permission
|
||||
if principal == owner {
|
||||
return true
|
||||
@@ -128,11 +126,8 @@ func checkPermission(operation, principal, owner, resourcePolicy, resourceARN st
|
||||
return true
|
||||
}
|
||||
|
||||
// If no policy is provided, use default allow if enabled
|
||||
// If no policy is provided, deny access (default deny)
|
||||
if resourcePolicy == "" {
|
||||
if ctx != nil && ctx.DefaultAllow {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -174,25 +169,15 @@ func checkPermission(operation, principal, owner, resourcePolicy, resourceARN st
|
||||
}
|
||||
|
||||
// Statement matches - check effect
|
||||
switch stmt.Effect {
|
||||
case "Allow":
|
||||
if stmt.Effect == "Allow" {
|
||||
hasAllow = true
|
||||
case "Deny":
|
||||
} else if stmt.Effect == "Deny" {
|
||||
// Explicit deny always wins
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if hasAllow {
|
||||
return true
|
||||
}
|
||||
|
||||
// If no statement matched, use default allow if enabled
|
||||
if ctx != nil && ctx.DefaultAllow {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
return hasAllow
|
||||
}
|
||||
|
||||
func hasIdentityPermission(operation string, ctx *PolicyContext) bool {
|
||||
|
||||
@@ -206,48 +206,3 @@ func TestEvaluatePolicyWithConditions(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestCheckPermissionWithDefaultAllow(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
defaultAllow bool
|
||||
policy string
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
"default deny (no policy, DefaultAllow=false)",
|
||||
false,
|
||||
"",
|
||||
false,
|
||||
},
|
||||
{
|
||||
"default allow (no policy, DefaultAllow=true)",
|
||||
true,
|
||||
"",
|
||||
true,
|
||||
},
|
||||
{
|
||||
"explicit deny overrides DefaultAllow=true",
|
||||
true,
|
||||
`{"Statement":[{"Effect":"Deny","Principal":"*","Action":"s3tables:GetTable"}]}`,
|
||||
false,
|
||||
},
|
||||
{
|
||||
"explicit allow works with DefaultAllow=false",
|
||||
false,
|
||||
`{"Statement":[{"Effect":"Allow","Principal":"*","Action":"s3tables:GetTable"}]}`,
|
||||
true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx := &PolicyContext{
|
||||
DefaultAllow: tt.defaultAllow,
|
||||
}
|
||||
result := CheckPermissionWithContext("s3tables:GetTable", "user123", "owner123", tt.policy, "", ctx)
|
||||
if result != tt.expected {
|
||||
t.Errorf("CheckPermissionWithContext() = %v, want %v (DefaultAllow=%v, Policy=%s)", result, tt.expected, tt.defaultAllow, tt.policy)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,9 +41,6 @@ func (m *mockIAMIntegration) ValidateTrustPolicyForPrincipal(ctx context.Context
|
||||
func (m *mockIAMIntegration) ValidateSessionToken(ctx context.Context, token string) (*sts.SessionInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *mockIAMIntegration) DefaultAllow() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func TestSTSAssumeRolePostBody(t *testing.T) {
|
||||
// Setup S3ApiServer with IAM enabled
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/maintenance"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/telemetry"
|
||||
|
||||
@@ -313,7 +312,6 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
|
||||
|
||||
func (ms *MasterServer) startAdminScripts() {
|
||||
v := util.GetViper()
|
||||
v.SetDefault("master.maintenance.scripts", maintenance.DefaultMasterMaintenanceScripts)
|
||||
adminScripts := v.GetString("master.maintenance.scripts")
|
||||
if adminScripts == "" {
|
||||
return
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
)
|
||||
|
||||
@@ -94,7 +95,7 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
|
||||
case volume_server_pb.VolumeScrubMode_LOCAL:
|
||||
files, shardInfos, serrs = v.ScrubLocal()
|
||||
case volume_server_pb.VolumeScrubMode_FULL:
|
||||
files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId)
|
||||
files, shardInfos, serrs = scrubEcVolumeFull(ctx, v)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported EC volume scrub mode %d", m)
|
||||
}
|
||||
@@ -119,3 +120,7 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func scrubEcVolumeFull(ctx context.Context, v *erasure_coding.EcVolume) (int64, []*volume_server_pb.EcShardInfo, []error) {
|
||||
return 0, nil, []error{fmt.Errorf("scrubEcVolumeFull(): not implemented, see https://github.com/seaweedfs/seaweedfs/issues/8018")}
|
||||
}
|
||||
|
||||
@@ -43,23 +43,6 @@ func AllShardIds() []ShardId {
|
||||
return res
|
||||
}
|
||||
|
||||
// Compares a pair of EcShardInfo protos for sorting.
|
||||
func CmpEcShardInfo(a, b *volume_server_pb.EcShardInfo) int {
|
||||
if a.VolumeId < b.VolumeId {
|
||||
return -1
|
||||
}
|
||||
if a.VolumeId > b.VolumeId {
|
||||
return 1
|
||||
}
|
||||
if a.ShardId < b.ShardId {
|
||||
return -1
|
||||
}
|
||||
if a.ShardId > b.ShardId {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type EcVolumeShard struct {
|
||||
VolumeId needle.VolumeId
|
||||
ShardId ShardId
|
||||
|
||||
+26
-45
@@ -159,7 +159,7 @@ func (s *Store) ReadEcShardNeedle(vid needle.VolumeId, n *needle.Needle, onReadS
|
||||
if len(intervals) > 1 {
|
||||
glog.V(3).Infof("ReadEcShardNeedle needle id %s intervals:%+v", n.String(), intervals)
|
||||
}
|
||||
bytes, isDeleted, err := s.readEcShardIntervals(n.Id, localEcVolume, intervals)
|
||||
bytes, isDeleted, err := s.readEcShardIntervals(vid, n.Id, localEcVolume, intervals)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("ReadEcShardIntervals: %w", err)
|
||||
}
|
||||
@@ -178,11 +178,8 @@ func (s *Store) ReadEcShardNeedle(vid needle.VolumeId, n *needle.Needle, onReadS
|
||||
return 0, fmt.Errorf("ec shard %d not found", vid)
|
||||
}
|
||||
|
||||
func (s *Store) IntervalToShardIdAndOffset(iv erasure_coding.Interval) (erasure_coding.ShardId, int64) {
|
||||
return iv.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
|
||||
}
|
||||
func (s *Store) readEcShardIntervals(vid needle.VolumeId, needleId types.NeedleId, ecVolume *erasure_coding.EcVolume, intervals []erasure_coding.Interval) (data []byte, is_deleted bool, err error) {
|
||||
|
||||
func (s *Store) readEcShardIntervals(needleId types.NeedleId, ecVolume *erasure_coding.EcVolume, intervals []erasure_coding.Interval) (data []byte, is_deleted bool, err error) {
|
||||
if err = s.cachedLookupEcShardLocations(ecVolume); err != nil {
|
||||
return nil, false, fmt.Errorf("failed to locate shard via master grpc %s: %v", s.MasterAddress, err)
|
||||
}
|
||||
@@ -205,36 +202,37 @@ func (s *Store) readEcShardIntervals(needleId types.NeedleId, ecVolume *erasure_
|
||||
}
|
||||
|
||||
func (s *Store) readOneEcShardInterval(needleId types.NeedleId, ecVolume *erasure_coding.EcVolume, interval erasure_coding.Interval) (data []byte, is_deleted bool, err error) {
|
||||
shardId, actualOffset := s.IntervalToShardIdAndOffset(interval)
|
||||
shardId, actualOffset := interval.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
|
||||
data = make([]byte, interval.Size)
|
||||
if shard, found := ecVolume.FindEcVolumeShard(shardId); found {
|
||||
var readSize int
|
||||
if readSize, err = shard.ReadAt(data, actualOffset); err != nil {
|
||||
if readSize != int(interval.Size) {
|
||||
glog.V(0).Infof("read local ec shard %d.%d offset %d: %v", ecVolume.VolumeId, shardId, actualOffset, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ecVolume.ShardLocationsLock.RLock()
|
||||
sourceDataNodes, hasShardIdLocation := ecVolume.ShardLocations[shardId]
|
||||
ecVolume.ShardLocationsLock.RUnlock()
|
||||
|
||||
// try local read
|
||||
err = s.readLocalEcShardInterval(ecVolume, shardId, data, actualOffset)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
glog.V(0).Infof("read local ec shard %d.%d offset %d: %v", ecVolume.VolumeId, shardId, actualOffset, err)
|
||||
// try reading directly
|
||||
if hasShardIdLocation {
|
||||
_, is_deleted, err = s.readRemoteEcShardInterval(sourceDataNodes, needleId, ecVolume.VolumeId, shardId, data, actualOffset)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
glog.V(0).Infof("clearing ec shard %d.%d locations: %v", ecVolume.VolumeId, shardId, err)
|
||||
}
|
||||
|
||||
ecVolume.ShardLocationsLock.RLock()
|
||||
sourceDataNodes, hasShardIdLocation := ecVolume.ShardLocations[shardId]
|
||||
ecVolume.ShardLocationsLock.RUnlock()
|
||||
|
||||
// try reading directly
|
||||
if hasShardIdLocation {
|
||||
_, is_deleted, err = s.readRemoteEcShardInterval(sourceDataNodes, needleId, ecVolume.VolumeId, shardId, data, actualOffset)
|
||||
// try reading by recovering from other shards
|
||||
_, is_deleted, err = s.recoverOneRemoteEcShardInterval(needleId, ecVolume, shardId, data, actualOffset)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
glog.V(0).Infof("read remote ec shard %d.%d locations: %v", ecVolume.VolumeId, shardId, err)
|
||||
glog.V(0).Infof("recover ec shard %d.%d : %v", ecVolume.VolumeId, shardId, err)
|
||||
}
|
||||
|
||||
// try reading by recovering from other shards
|
||||
_, is_deleted, err = s.recoverOneRemoteEcShardInterval(needleId, ecVolume, shardId, data, actualOffset)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
glog.V(0).Infof("recover ec shard %d.%d : %v", ecVolume.VolumeId, shardId, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -288,23 +286,6 @@ func (s *Store) cachedLookupEcShardLocations(ecVolume *erasure_coding.EcVolume)
|
||||
return
|
||||
}
|
||||
|
||||
func (s *Store) readLocalEcShardInterval(ecVolume *erasure_coding.EcVolume, shardId erasure_coding.ShardId, buf []byte, offset int64) error {
|
||||
shard, found := ecVolume.FindEcVolumeShard(shardId)
|
||||
if !found {
|
||||
return fmt.Errorf("shard %d not found for volume %d", shardId, ecVolume.VolumeId)
|
||||
}
|
||||
|
||||
readBytes, err := shard.ReadAt(buf, offset)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read local EC shard %d for volume %d: %v", shardId, ecVolume.VolumeId, err)
|
||||
}
|
||||
if got, want := readBytes, len(buf); got != want {
|
||||
return fmt.Errorf("expected %d bytes for local EC shard %d on volume %d, got %d", want, shardId, ecVolume.VolumeId, got)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) readRemoteEcShardInterval(sourceDataNodes []pb.ServerAddress, needleId types.NeedleId, vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, is_deleted bool, err error) {
|
||||
|
||||
if len(sourceDataNodes) == 0 {
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
// ScrubEcVolume checks the full integrity of a EC volume, across both local and remote shards.
|
||||
// Returns a count of processed file entries, slice of found broken shards, and slice of found errors.
|
||||
func (s *Store) ScrubEcVolume(vid needle.VolumeId) (int64, []*volume_server_pb.EcShardInfo, []error) {
|
||||
ecv, found := s.FindEcVolume(vid)
|
||||
if !found {
|
||||
return 0, nil, []error{fmt.Errorf("EC volume id %d not found", vid)}
|
||||
}
|
||||
if err := s.cachedLookupEcShardLocations(ecv); err != nil {
|
||||
return 0, nil, []error{fmt.Errorf("failed to locate shard via master grpc %s: %v", s.MasterAddress, err)}
|
||||
}
|
||||
|
||||
// full scan means verifying indexes as well
|
||||
_, errs := ecv.ScrubIndex()
|
||||
|
||||
var count int64
|
||||
// reads for EC chunks can hit the same shard multiple times, so dedupe upon read errors
|
||||
brokenShardsMap := map[erasure_coding.ShardId]*volume_server_pb.EcShardInfo{}
|
||||
|
||||
err := ecv.WalkIndex(func(id types.NeedleId, offset types.Offset, size types.Size) error {
|
||||
count++
|
||||
if size.IsTombstone() {
|
||||
// nothing to do for tombstones...
|
||||
return nil
|
||||
}
|
||||
|
||||
data := make([]byte, 0, needle.GetActualSize(size, ecv.Version))
|
||||
intervals := ecv.LocateEcShardNeedleInterval(ecv.Version, offset.ToActualOffset(), size)
|
||||
|
||||
for i, iv := range intervals {
|
||||
chunk := make([]byte, iv.Size)
|
||||
shardId, offset := s.IntervalToShardIdAndOffset(iv)
|
||||
|
||||
// try a local shard read first...
|
||||
if err := s.readLocalEcShardInterval(ecv, shardId, chunk, offset); err == nil {
|
||||
data = append(data, chunk...)
|
||||
continue
|
||||
}
|
||||
|
||||
// ...then remote. note we do not try to recover EC-encoded data upon read failures;
|
||||
// we want check that shards are valid without decoding
|
||||
ecv.ShardLocationsLock.RLock()
|
||||
sourceDataNodes, ok := ecv.ShardLocations[shardId]
|
||||
ecv.ShardLocationsLock.RUnlock()
|
||||
if ok {
|
||||
if _, _, err := s.readRemoteEcShardInterval(sourceDataNodes, id, ecv.VolumeId, shardId, chunk, offset); err == nil {
|
||||
data = append(data, chunk...)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// chunk read for shard failed :(
|
||||
errs = append(errs, fmt.Errorf("failed to read EC shard %d for needle %d on volume %d (interval %d/%d)", shardId, id, ecv.VolumeId, i+1, len(intervals)))
|
||||
brokenShardsMap[shardId] = &volume_server_pb.EcShardInfo{
|
||||
ShardId: uint32(shardId),
|
||||
Size: int64(iv.Size),
|
||||
Collection: ecv.Collection,
|
||||
VolumeId: uint32(ecv.VolumeId),
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if got, want := int64(len(data)), needle.GetActualSize(size, ecv.Version); got != want {
|
||||
errs = append(errs, fmt.Errorf("expected %d bytes for needle %d, got %d", want, id, got))
|
||||
return nil
|
||||
}
|
||||
|
||||
n := needle.Needle{}
|
||||
if err := n.ReadBytes(data, 0, size, ecv.Version); err != nil {
|
||||
errs = append(errs, fmt.Errorf("needle %d on EC volume %d: %v", id, ecv.VolumeId, err))
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
brokenShards := []*volume_server_pb.EcShardInfo{}
|
||||
for _, s := range brokenShardsMap {
|
||||
brokenShards = append(brokenShards, s)
|
||||
}
|
||||
slices.SortFunc(brokenShards, erasure_coding.CmpEcShardInfo)
|
||||
|
||||
return count, brokenShards, errs
|
||||
}
|
||||
@@ -16,7 +16,6 @@ func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) {
|
||||
}
|
||||
return 0, fmt.Errorf("volume id %d is not found during check compact", volumeId)
|
||||
}
|
||||
|
||||
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
|
||||
if v := s.findVolume(vid); v != nil {
|
||||
// Get current volume size for space calculation
|
||||
@@ -40,15 +39,10 @@ func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compaction
|
||||
glog.V(1).Infof("volume %d compaction space check: volume=%d, index=%d, space_needed=%d, free_space=%d",
|
||||
vid, volumeSize, indexSize, spaceNeeded, diskStatus.Free)
|
||||
|
||||
return v.CompactByIndex(&CompactOptions{
|
||||
PreallocateBytes: preallocate,
|
||||
MaxBytesPerSecond: compactionBytePerSecond,
|
||||
ProgressCallback: progressFn,
|
||||
})
|
||||
return v.Compact2(preallocate, compactionBytePerSecond, progressFn)
|
||||
}
|
||||
return fmt.Errorf("volume id %d is not found during compact", vid)
|
||||
}
|
||||
|
||||
func (s *Store) CommitCompactVolume(vid needle.VolumeId) (bool, int64, error) {
|
||||
if s.isStopping {
|
||||
return false, 0, fmt.Errorf("volume id %d skips compact because volume is stopping", vid)
|
||||
@@ -64,7 +58,6 @@ func (s *Store) CommitCompactVolume(vid needle.VolumeId) (bool, int64, error) {
|
||||
}
|
||||
return false, 0, fmt.Errorf("volume id %d is not found during commit compact", vid)
|
||||
}
|
||||
|
||||
func (s *Store) CommitCleanupVolume(vid needle.VolumeId) error {
|
||||
if v := s.findVolume(vid); v != nil {
|
||||
return v.cleanupCompact()
|
||||
|
||||
@@ -35,26 +35,8 @@ func (v *Volume) garbageLevel() float64 {
|
||||
return float64(deletedSize) / float64(fileSize)
|
||||
}
|
||||
|
||||
type CompactOptions struct {
|
||||
PreallocateBytes int64
|
||||
MaxBytesPerSecond int64
|
||||
ProgressCallback ProgressFunc
|
||||
|
||||
// internal state settings
|
||||
srcDatPath string
|
||||
srcIdxPath string
|
||||
destDatPath string
|
||||
destIdxPath string
|
||||
superBlock super_block.SuperBlock
|
||||
version needle.Version
|
||||
}
|
||||
|
||||
// compact a volume based on deletions in .dat files
|
||||
func (v *Volume) CompactByVolumeData(opts *CompactOptions) error {
|
||||
if opts == nil {
|
||||
// default settings
|
||||
opts = &CompactOptions{}
|
||||
}
|
||||
func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error {
|
||||
|
||||
if v.MemoryMapMaxSizeMb != 0 { //it makes no sense to compact in memory
|
||||
return nil
|
||||
@@ -82,20 +64,11 @@ func (v *Volume) CompactByVolumeData(opts *CompactOptions) error {
|
||||
if err := v.nm.Sync(); err != nil {
|
||||
glog.V(0).Infof("compact failed to sync volume idx %d", v.Id)
|
||||
}
|
||||
|
||||
opts.destDatPath = v.FileName(".cpd")
|
||||
opts.destIdxPath = v.FileName(".cpx")
|
||||
opts.superBlock = v.SuperBlock
|
||||
opts.version = v.Version()
|
||||
return v.copyDataAndGenerateIndexFile(opts)
|
||||
return v.copyDataAndGenerateIndexFile(v.FileName(".cpd"), v.FileName(".cpx"), preallocate, compactionBytePerSecond)
|
||||
}
|
||||
|
||||
// compact a volume based on deletions in .idx files
|
||||
func (v *Volume) CompactByIndex(opts *CompactOptions) error {
|
||||
if opts == nil {
|
||||
// default settings
|
||||
opts = &CompactOptions{}
|
||||
}
|
||||
func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
|
||||
|
||||
if v.MemoryMapMaxSizeMb != 0 { //it makes no sense to compact in memory
|
||||
return nil
|
||||
@@ -123,14 +96,15 @@ func (v *Volume) CompactByIndex(opts *CompactOptions) error {
|
||||
if err := v.nm.Sync(); err != nil {
|
||||
glog.V(0).Infof("compact2 failed to sync volume idx %d: %v", v.Id, err)
|
||||
}
|
||||
|
||||
opts.srcDatPath = v.FileName(".dat")
|
||||
opts.srcIdxPath = v.FileName(".idx")
|
||||
opts.destDatPath = v.FileName(".cpd")
|
||||
opts.destIdxPath = v.FileName(".cpx")
|
||||
opts.superBlock = v.SuperBlock
|
||||
opts.version = v.Version()
|
||||
return v.copyDataBasedOnIndexFile(opts)
|
||||
return v.copyDataBasedOnIndexFile(
|
||||
v.FileName(".dat"), v.FileName(".idx"),
|
||||
v.FileName(".cpd"), v.FileName(".cpx"),
|
||||
v.SuperBlock,
|
||||
v.Version(),
|
||||
preallocate,
|
||||
compactionBytePerSecond,
|
||||
progressFn,
|
||||
)
|
||||
}
|
||||
|
||||
func (v *Volume) CommitCompact() error {
|
||||
@@ -429,9 +403,9 @@ func (scanner *VolumeFileScanner4Vacuum) VisitNeedle(n *needle.Needle, offset in
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *Volume) copyDataAndGenerateIndexFile(opts *CompactOptions) (err error) {
|
||||
func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, preallocate int64, compactionBytePerSecond int64) (err error) {
|
||||
var dst backend.BackendStorageFile
|
||||
if dst, err = backend.CreateVolumeFile(opts.destDatPath, opts.PreallocateBytes, 0); err != nil {
|
||||
if dst, err = backend.CreateVolumeFile(dstName, preallocate, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
defer dst.Close()
|
||||
@@ -444,7 +418,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(opts *CompactOptions) (err error)
|
||||
now: uint64(time.Now().Unix()),
|
||||
nm: nm,
|
||||
dstBackend: dst,
|
||||
writeThrottler: util.NewWriteThrottler(opts.MaxBytesPerSecond),
|
||||
writeThrottler: util.NewWriteThrottler(compactionBytePerSecond),
|
||||
}
|
||||
err = ScanVolumeFile(v.dir, v.Collection, v.Id, v.needleMapKind, scanner)
|
||||
if err != nil {
|
||||
@@ -452,15 +426,15 @@ func (v *Volume) copyDataAndGenerateIndexFile(opts *CompactOptions) (err error)
|
||||
return err
|
||||
}
|
||||
|
||||
return nm.SaveToIdx(opts.destIdxPath)
|
||||
return nm.SaveToIdx(idxName)
|
||||
}
|
||||
|
||||
func (v *Volume) copyDataBasedOnIndexFile(opts *CompactOptions) (err error) {
|
||||
func (v *Volume) copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName string, sb super_block.SuperBlock, version needle.Version, preallocate, compactionBytePerSecond int64, progressFn ProgressFunc) (err error) {
|
||||
var (
|
||||
srcDatBackend, dstDatBackend backend.BackendStorageFile
|
||||
dataFile *os.File
|
||||
)
|
||||
if dstDatBackend, err = backend.CreateVolumeFile(opts.destDatPath, opts.PreallocateBytes, 0); err != nil {
|
||||
if dstDatBackend, err = backend.CreateVolumeFile(dstDatName, preallocate, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
@@ -472,10 +446,10 @@ func (v *Volume) copyDataBasedOnIndexFile(opts *CompactOptions) (err error) {
|
||||
defer oldNm.Close()
|
||||
newNm := needle_map.NewMemDb()
|
||||
defer newNm.Close()
|
||||
if err = oldNm.LoadFromIdx(opts.srcIdxPath); err != nil {
|
||||
if err = oldNm.LoadFromIdx(srcIdxName); err != nil {
|
||||
return err
|
||||
}
|
||||
if dataFile, err = os.Open(opts.srcDatPath); err != nil {
|
||||
if dataFile, err = os.Open(srcDatName); err != nil {
|
||||
return err
|
||||
}
|
||||
srcDatBackend = backend.NewDiskFile(dataFile)
|
||||
@@ -483,13 +457,11 @@ func (v *Volume) copyDataBasedOnIndexFile(opts *CompactOptions) (err error) {
|
||||
|
||||
now := uint64(time.Now().Unix())
|
||||
|
||||
opts.superBlock.CompactionRevision++
|
||||
newOffset := int64(opts.superBlock.BlockSize())
|
||||
if _, err := dstDatBackend.WriteAt(opts.superBlock.Bytes(), 0); err != nil {
|
||||
return fmt.Errorf("failed to write superblock: %v", err)
|
||||
}
|
||||
sb.CompactionRevision++
|
||||
dstDatBackend.WriteAt(sb.Bytes(), 0)
|
||||
newOffset := int64(sb.BlockSize())
|
||||
|
||||
writeThrottler := util.NewWriteThrottler(opts.MaxBytesPerSecond)
|
||||
writeThrottler := util.NewWriteThrottler(compactionBytePerSecond)
|
||||
err = oldNm.AscendingVisit(func(value needle_map.NeedleValue) error {
|
||||
|
||||
offset, size := value.Offset, value.Size
|
||||
@@ -498,29 +470,29 @@ func (v *Volume) copyDataBasedOnIndexFile(opts *CompactOptions) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if opts.ProgressCallback != nil {
|
||||
if !opts.ProgressCallback(offset.ToActualOffset()) {
|
||||
if progressFn != nil {
|
||||
if !progressFn(offset.ToActualOffset()) {
|
||||
return fmt.Errorf("interrupted")
|
||||
}
|
||||
}
|
||||
|
||||
n := new(needle.Needle)
|
||||
if err := n.ReadData(srcDatBackend, offset.ToActualOffset(), size, opts.version); err != nil {
|
||||
if err := n.ReadData(srcDatBackend, offset.ToActualOffset(), size, version); err != nil {
|
||||
v.checkReadWriteError(err)
|
||||
return fmt.Errorf("cannot hydrate needle from file: %w", err)
|
||||
}
|
||||
|
||||
if n.HasTtl() && now >= n.LastModified+uint64(opts.superBlock.Ttl.Minutes()*60) {
|
||||
if n.HasTtl() && now >= n.LastModified+uint64(sb.Ttl.Minutes()*60) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = newNm.Set(n.Id, ToOffset(newOffset), n.Size); err != nil {
|
||||
return fmt.Errorf("cannot put needle: %s", err)
|
||||
}
|
||||
if _, _, _, err = n.Append(dstDatBackend, opts.superBlock.Version); err != nil {
|
||||
if _, _, _, err = n.Append(dstDatBackend, sb.Version); err != nil {
|
||||
return fmt.Errorf("cannot append needle: %s", err)
|
||||
}
|
||||
delta := n.DiskSize(opts.version)
|
||||
delta := n.DiskSize(version)
|
||||
newOffset += delta
|
||||
writeThrottler.MaybeSlowdown(delta)
|
||||
glog.V(4).Infoln("saving key", n.Id, "volume offset", offset, "=>", newOffset, "data_size", n.Size)
|
||||
@@ -546,14 +518,14 @@ func (v *Volume) copyDataBasedOnIndexFile(opts *CompactOptions) (err error) {
|
||||
v.Id.String(), v.nm.ContentSize(), v.nm.DeletedSize(), dstDatSize)
|
||||
}
|
||||
}
|
||||
err = newNm.SaveToIdx(opts.destIdxPath)
|
||||
err = newNm.SaveToIdx(datIdxName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
indexFile, err := os.OpenFile(opts.destIdxPath, os.O_RDWR|os.O_CREATE, 0644)
|
||||
indexFile, err := os.OpenFile(datIdxName, os.O_RDWR|os.O_CREATE, 0644)
|
||||
if err != nil {
|
||||
glog.Errorf("cannot open Volume Index %s: %v", opts.destIdxPath, err)
|
||||
glog.Errorf("cannot open Volume Index %s: %v", datIdxName, err)
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
|
||||
@@ -62,14 +62,14 @@ func TestMakeDiff(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMemIndexCompaction(t *testing.T) {
|
||||
testCompactionByIndex(t, NeedleMapInMemory)
|
||||
testCompaction(t, NeedleMapInMemory)
|
||||
}
|
||||
|
||||
func TestLDBIndexCompaction(t *testing.T) {
|
||||
testCompactionByIndex(t, NeedleMapLevelDb)
|
||||
testCompaction(t, NeedleMapLevelDb)
|
||||
}
|
||||
|
||||
func testCompactionByIndex(t *testing.T, needleMapKind NeedleMapKind) {
|
||||
func testCompaction(t *testing.T, needleMapKind NeedleMapKind) {
|
||||
dir := t.TempDir()
|
||||
|
||||
v, err := NewVolume(dir, dir, "", 1, needleMapKind, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
|
||||
@@ -87,7 +87,7 @@ func testCompactionByIndex(t *testing.T, needleMapKind NeedleMapKind) {
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
v.CompactByIndex(nil)
|
||||
v.Compact2(0, 0, nil)
|
||||
speed := float64(v.ContentSize()) / time.Now().Sub(startTime).Seconds()
|
||||
t.Logf("compaction speed: %.2f bytes/s", speed)
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
var (
|
||||
MAJOR_VERSION = int32(4)
|
||||
MINOR_VERSION = int32(13)
|
||||
MINOR_VERSION = int32(12)
|
||||
VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION)
|
||||
VERSION = util.SizeLimit + " " + VERSION_NUMBER
|
||||
COMMIT = ""
|
||||
|
||||
Reference in New Issue
Block a user