Compare commits

..
Author SHA1 Message Date
chrislu e7d12a1c04 rename 2025-07-19 00:46:35 -07:00
chrislu 11fb65571f refactor 2025-07-19 00:42:43 -07:00
chrislu 6c2a11f1d6 Merge branch 'master' into fix-GetObjectLockConfigurationHandler 2025-07-19 00:42:36 -07:00
chrislu 851011a32b address comments 2025-07-18 22:16:15 -07:00
chrislu a9a9c732b9 reduce logs 2025-07-18 21:51:13 -07:00
6328141019 Update weed/s3api/s3api_object_retention.go
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-18 21:46:59 -07:00
chrislu b508e00de7 fix test 2025-07-18 21:40:04 -07:00
chrislu e486817499 fix tests 2025-07-18 21:32:38 -07:00
chrislu aec8542b6e fix test itself 2025-07-18 21:17:42 -07:00
chrislu 8fb9df72ae fix tests 2025-07-18 20:55:12 -07:00
chrislu af25f92e36 fix tests 2025-07-18 20:52:03 -07:00
chrislu 66829fa7c8 fix tests 2025-07-18 20:46:56 -07:00
chrislu 5c24bfc795 fix test_object_lock_delete_object_with_retention_and_marker 2025-07-18 20:43:09 -07:00
chrislu 1d1c717493 refactor 2025-07-18 20:16:04 -07:00
chrislu 7661449f82 fix tests 2025-07-18 20:10:16 -07:00
chrislu a5a6b5a4d4 fix test_object_lock_delete_multipart_object_with_legal_hold_on 2025-07-18 19:52:51 -07:00
chrislu 8121fdeec3 fix many issues 2025-07-18 19:40:12 -07:00
chrislu f63f40a3d4 fixes 2025-07-18 18:04:10 -07:00
chrislu 2d6532d80e fix test_object_lock_put_obj_lock_invalid_days 2025-07-18 17:51:14 -07:00
chrislu dce0672ae0 Update s3tests.conf 2025-07-18 17:19:15 -07:00
chrislu 40d9f4fa9c add error mapping 2025-07-18 17:08:59 -07:00
chrislu 8cc99ca121 fixes 2025-07-18 17:06:46 -07:00
chrislu c5628b000b fix tests 2025-07-18 16:47:25 -07:00
chrislu f419271299 pass tests 2025-07-18 16:38:24 -07:00
chrislu a0ab227e07 fixes 2025-07-18 16:25:45 -07:00
chrislu 2ad4a924be fix tests 2025-07-18 16:23:15 -07:00
chrislu 1e1806c7a1 fixes
✅ Include VersionId in multipart upload completion responses when versioning is enabled
✅ Block retention mode changes (GOVERNANCE ↔ COMPLIANCE) without bypass permissions
✅ Handle all object lock validation errors consistently with proper error codes
✅ Pass the remaining object lock tests
2025-07-18 16:17:36 -07:00
chrislu eaaef569ce fixes
✅ Return 409 (Conflict) with InvalidBucketState for bucket-level object lock configuration operations on buckets without object lock enabled
✅ Allow increasing retention periods and overriding retention with same/later dates
✅ Only block decreasing retention periods without proper bypass permissions
✅ Handle all object lock validation errors consistently with proper error codes
2025-07-18 16:11:14 -07:00
chrislu 1807f4e94d fixes
✅ Return MalformedXML when both Days and Years are specified in the same retention configuration
✅ Return 400 (Bad Request) with InvalidRequest when object lock operations are attempted on buckets without object lock enabled
✅ Handle all object lock validation errors consistently with proper error codes
2025-07-18 16:03:06 -07:00
chrislu 7203c78e4d constants and fixes
✅ Return InvalidRetentionPeriod for invalid retention values (0 days, negative years)
✅ Return ObjectLockConfigurationNotFoundError when object lock configuration doesn't exist
✅ Handle all object lock validation errors consistently with proper error codes
2025-07-18 15:58:07 -07:00
chrislu 1bc503530a fixes
With these final fixes, the s3-tests should now:
✅ Return MalformedXML for ObjectLockEnabled: 'Disabled'
✅ Return MalformedXML when both Days and Years are specified in retention configuration
✅ Return InvalidBucketState (409 Conflict) when trying to suspend versioning on buckets with object lock enabled
✅ Handle all object lock validation errors consistently with proper error codes
2025-07-18 15:52:28 -07:00
chrislu 9f88fd2ea5 fixes
With these comprehensive fixes, the s3-tests should now:
✅ Return InvalidBucketState (409 Conflict) for object lock operations on invalid buckets
✅ Return InvalidRetentionPeriod for invalid retention periods
✅ Return MalformedXML for malformed retention configurations
✅ Include VersionId in response headers when available
✅ Return proper HTTP status codes for all error conditions
✅ Handle all object lock validation errors consistently
The workflow should now pass significantly more object lock tests, bringing SeaweedFS's S3 object lock implementation much closer to AWS S3 compatibility standards.
2025-07-18 15:45:15 -07:00
chrislu 0123abe49b address test errors
With these fixes, the s3-tests should now:
✅ Return InvalidBucketState (409 Conflict) for object lock operations on invalid buckets
✅ Return MalformedXML for invalid retention configurations
✅ Include VersionId in response headers when available
✅ Return proper HTTP status codes (403 Forbidden for retention mode changes)
✅ Handle all object lock validation errors consistently
2025-07-18 15:39:30 -07:00
chrislu 4b92f03b65 address test errors 2025-07-18 15:33:44 -07:00
chrislu 1e7a3c09d4 Update s3tests.conf 2025-07-18 15:23:05 -07:00
chrislu ddb2105f00 Update s3tests.conf 2025-07-18 15:17:58 -07:00
chrislu bbcd19f6c6 Update s3tests.yml 2025-07-18 15:14:09 -07:00
chrislu fc2fcae543 Update s3tests.yml 2025-07-18 15:03:21 -07:00
chrislu 4801f641f1 install s3 tests 2025-07-18 15:01:25 -07:00
chrislu 770888df81 use marker 2025-07-18 14:54:15 -07:00
chrislu 1dfdd3c84d Merge branch 'master' into fix-GetObjectLockConfigurationHandler 2025-07-18 14:47:36 -07:00
chrislu b63855560a add s3 tests for object lock and retention 2025-07-18 14:46:32 -07:00
chrislu 9f8e0257b0 read entry once 2025-07-18 14:39:02 -07:00
chrislu e53cd29eb2 errorf 2025-07-18 02:14:19 -07:00
chrislu eeeb595346 change error style 2025-07-18 02:02:54 -07:00
678f962ded Update weed/s3api/object_lock_utils.go
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-18 01:55:43 -07:00
a906358c59 Update test/s3/retention/s3_retention_test.go
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-18 01:54:36 -07:00
12f34f965f Update weed/s3api/s3api_object_handlers_put.go
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-07-18 01:54:14 -07:00
chrislu 1db149ecc2 address comments 2025-07-18 01:49:44 -07:00
chrislu 59f8444a90 fix test 2025-07-18 01:42:03 -07:00
chrislu bcc7a74a25 use fields instead of xml 2025-07-18 01:39:23 -07:00
chrislu 0d4eeb67be admin ui and api handler are consistent now 2025-07-18 01:27:46 -07:00
chrislu 85e34df214 fix routing 2025-07-18 01:27:27 -07:00
chrislu bb05e6cade Update s3api_object_handlers_put.go 2025-07-18 00:53:20 -07:00
chrislu bc46214445 rename variable 2025-07-18 00:52:38 -07:00
chrislu 121f6ef2c0 avoid duplidated work 2025-07-18 00:48:56 -07:00
Chris LuGitHubgemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
22673fee76 Update weed/s3api/s3api_server.go
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
2025-07-18 00:47:24 -07:00
chrislu 3fbab20b83 refactor 2025-07-18 00:34:16 -07:00
chrislu b8cbe9b6b2 increase bucket config cache TTL 2025-07-18 00:26:58 -07:00
chrislu 8bb47f3c03 subscribe to bucket configuration changes 2025-07-18 00:25:18 -07:00
chrislu 7bec2ef94b cache and use bucket object lock config 2025-07-18 00:19:47 -07:00
chrislu f9cbed7ee1 fix GetObjectLockConfigurationHandler 2025-07-17 23:12:19 -07:00
29 changed files with 346 additions and 3493 deletions
+29 -607
View File
@@ -14,10 +14,10 @@ permissions:
contents: read
jobs:
basic-s3-tests:
name: Basic S3 tests (KV store)
s3tests:
name: Ceph S3 tests
runs-on: ubuntu-22.04
timeout-minutes: 15
timeout-minutes: 30
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
@@ -41,7 +41,7 @@ jobs:
pip install tox
pip install -e .
- name: Run Basic S3 tests
- name: Run Ceph S3 tests with KV store
timeout-minutes: 15
env:
S3TEST_CONF: ../docker/compose/s3tests.conf
@@ -55,76 +55,13 @@ jobs:
mkdir -p "$WEED_DATA_DIR"
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9333 -volume.port=8080 -filer.port=8888 -s3.port=8000 -metricsPort=9324 \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
pid=$!
# Wait for all SeaweedFS components to be ready
echo "Waiting for SeaweedFS components to start..."
for i in {1..30}; do
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then
echo "Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8080/status > /dev/null 2>&1; then
echo "Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8888/ > /dev/null 2>&1; then
echo "Filer is ready"
break
fi
echo "Waiting for filer... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8000/ > /dev/null 2>&1; then
echo "S3 server is ready"
break
fi
echo "Waiting for S3 server... ($i/30)"
sleep 2
done
echo "All SeaweedFS components are ready!"
sleep 10
cd ../s3-tests
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
# Debug: Show the config file contents
echo "=== S3 Config File Contents ==="
cat ../docker/compose/s3tests.conf
echo "=== End Config ==="
# Additional wait for S3-Filer integration to be fully ready
echo "Waiting additional 10 seconds for S3-Filer integration..."
sleep 10
# Test S3 connection before running tests
echo "Testing S3 connection..."
for i in {1..10}; do
if curl -s -f http://localhost:8000/ > /dev/null 2>&1; then
echo "S3 connection test successful"
break
fi
echo "S3 connection test failed, retrying... ($i/10)"
sleep 2
done
echo "✅ S3 server is responding, starting tests..."
tox -- \
s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
@@ -283,265 +220,33 @@ jobs:
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
versioning-tests:
name: S3 Versioning & Object Lock tests
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go 1.x
uses: actions/setup-go@v5.5.0
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Clone s3-tests
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Run S3 Object Lock, Retention, and Versioning tests
- name: Run S3 Object Lock and Retention tests
timeout-minutes: 15
env:
S3TEST_CONF: ../docker/compose/s3tests.conf
shell: bash
run: |
cd weed
go install -buildvcs=false
set -x
# Create clean data directory for this test run
export WEED_DATA_DIR="/tmp/seaweedfs-objectlock-versioning-$(date +%s)"
export WEED_DATA_DIR="/tmp/seaweedfs-objectlock-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9334 -volume.port=8081 -filer.port=8889 -s3.port=8001 -metricsPort=9325 \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
pid=$!
# Wait for all SeaweedFS components to be ready
echo "Waiting for SeaweedFS components to start..."
for i in {1..30}; do
if curl -s http://localhost:9334/cluster/status > /dev/null 2>&1; then
echo "Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8081/status > /dev/null 2>&1; then
echo "Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8889/ > /dev/null 2>&1; then
echo "Filer is ready"
break
fi
echo "Waiting for filer... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8001/ > /dev/null 2>&1; then
echo "S3 server is ready"
break
fi
echo "Waiting for S3 server... ($i/30)"
sleep 2
done
echo "All SeaweedFS components are ready!"
sleep 10
cd ../s3-tests
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
# Fix bucket creation conflicts in versioning tests by replacing _create_objects calls
sed -i 's/bucket_name = _create_objects(bucket_name=bucket_name,keys=key_names)/# Use the existing bucket for object creation\n client = get_client()\n for key in key_names:\n client.put_object(Bucket=bucket_name, Body=key, Key=key)/' s3tests_boto3/functional/test_s3.py
sed -i 's/bucket = _create_objects(bucket_name=bucket_name, keys=key_names)/# Use the existing bucket for object creation\n client = get_client()\n for key in key_names:\n client.put_object(Bucket=bucket_name, Body=key, Key=key)/' s3tests_boto3/functional/test_s3.py
# Create and update s3tests.conf to use port 8001
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-versioning.conf
sed -i 's/port = 8000/port = 8001/g' ../docker/compose/s3tests-versioning.conf
sed -i 's/:8000/:8001/g' ../docker/compose/s3tests-versioning.conf
sed -i 's/localhost:8000/localhost:8001/g' ../docker/compose/s3tests-versioning.conf
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8001/g' ../docker/compose/s3tests-versioning.conf
export S3TEST_CONF=../docker/compose/s3tests-versioning.conf
# Debug: Show the config file contents
echo "=== S3 Config File Contents ==="
cat ../docker/compose/s3tests-versioning.conf
echo "=== End Config ==="
# Additional wait for S3-Filer integration to be fully ready
echo "Waiting additional 10 seconds for S3-Filer integration..."
sleep 10
# Test S3 connection before running tests
echo "Testing S3 connection..."
for i in {1..10}; do
if curl -s -f http://localhost:8001/ > /dev/null 2>&1; then
echo "S3 connection test successful"
break
fi
echo "S3 connection test failed, retrying... ($i/10)"
sleep 2
done
# tox -- s3tests_boto3/functional/test_s3.py -k "object_lock or (versioning and not test_versioning_obj_suspend_versions and not test_bucket_list_return_data_versioning and not test_versioning_concurrent_multi_object_delete)" --tb=short
tox -- s3tests_boto3/functional/test_s3.py -k "object_lock or versioning" --tb=short
# Run object lock tests by pattern matching test names
tox -- -k "object_lock" --tb=short
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
cors-tests:
name: S3 CORS tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go 1.x
uses: actions/setup-go@v5.5.0
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Clone s3-tests
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Run S3 CORS tests
timeout-minutes: 10
shell: bash
run: |
cd weed
go install -buildvcs=false
set -x
# Create clean data directory for this test run
export WEED_DATA_DIR="/tmp/seaweedfs-cors-test-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9335 -volume.port=8082 -filer.port=8890 -s3.port=8002 -metricsPort=9326 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
pid=$!
# Wait for all SeaweedFS components to be ready
echo "Waiting for SeaweedFS components to start..."
for i in {1..30}; do
if curl -s http://localhost:9335/cluster/status > /dev/null 2>&1; then
echo "Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8082/status > /dev/null 2>&1; then
echo "Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8890/ > /dev/null 2>&1; then
echo "Filer is ready"
break
fi
echo "Waiting for filer... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8002/ > /dev/null 2>&1; then
echo "S3 server is ready"
break
fi
echo "Waiting for S3 server... ($i/30)"
sleep 2
done
echo "All SeaweedFS components are ready!"
cd ../s3-tests
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
# Create and update s3tests.conf to use port 8002
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-cors.conf
sed -i 's/port = 8000/port = 8002/g' ../docker/compose/s3tests-cors.conf
sed -i 's/:8000/:8002/g' ../docker/compose/s3tests-cors.conf
sed -i 's/localhost:8000/localhost:8002/g' ../docker/compose/s3tests-cors.conf
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8002/g' ../docker/compose/s3tests-cors.conf
export S3TEST_CONF=../docker/compose/s3tests-cors.conf
# Debug: Show the config file contents
echo "=== S3 Config File Contents ==="
cat ../docker/compose/s3tests-cors.conf
echo "=== End Config ==="
# Additional wait for S3-Filer integration to be fully ready
echo "Waiting additional 10 seconds for S3-Filer integration..."
sleep 10
# Test S3 connection before running tests
echo "Testing S3 connection..."
for i in {1..10}; do
if curl -s -f http://localhost:8002/ > /dev/null 2>&1; then
echo "S3 connection test successful"
break
fi
echo "S3 connection test failed, retrying... ($i/10)"
sleep 2
done
# Run CORS-specific tests from s3-tests suite
tox -- s3tests_boto3/functional/test_s3.py -k "cors" --tb=short || echo "No CORS tests found in s3-tests suite"
# If no specific CORS tests exist, run bucket configuration tests that include CORS
tox -- s3tests_boto3/functional/test_s3.py::test_put_bucket_cors || echo "No put_bucket_cors test found"
tox -- s3tests_boto3/functional/test_s3.py::test_get_bucket_cors || echo "No get_bucket_cors test found"
tox -- s3tests_boto3/functional/test_s3.py::test_delete_bucket_cors || echo "No delete_bucket_cors test found"
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
copy-tests:
name: SeaweedFS Custom S3 Copy tests
runs-on: ubuntu-22.04
timeout-minutes: 10
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go 1.x
uses: actions/setup-go@v5.5.0
with:
go-version-file: 'go.mod'
id: go
- name: Run SeaweedFS Custom S3 Copy tests
timeout-minutes: 10
shell: bash
@@ -554,322 +259,39 @@ jobs:
set -x
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
pid=$!
# Wait for all SeaweedFS components to be ready
echo "Waiting for SeaweedFS components to start..."
for i in {1..30}; do
if curl -s http://localhost:9336/cluster/status > /dev/null 2>&1; then
echo "Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8083/status > /dev/null 2>&1; then
echo "Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8891/ > /dev/null 2>&1; then
echo "Filer is ready"
break
fi
echo "Waiting for filer... ($i/30)"
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8003/ > /dev/null 2>&1; then
echo "S3 server is ready"
break
fi
echo "Waiting for S3 server... ($i/30)"
sleep 2
done
echo "All SeaweedFS components are ready!"
cd ../test/s3/copying
# Patch Go tests to use the correct S3 endpoint (port 8003)
sed -i 's/http:\/\/127\.0\.0\.1:8000/http:\/\/127.0.0.1:8003/g' s3_copying_test.go
# Debug: Show what endpoint the Go tests will use
echo "=== Go Test Configuration ==="
grep -n "127.0.0.1" s3_copying_test.go || echo "No IP configuration found"
echo "=== End Configuration ==="
# Additional wait for S3-Filer integration to be fully ready
echo "Waiting additional 10 seconds for S3-Filer integration..."
sleep 10
# Test S3 connection before running tests
echo "Testing S3 connection..."
for i in {1..10}; do
if curl -s -f http://localhost:8003/ > /dev/null 2>&1; then
echo "S3 connection test successful"
break
fi
echo "S3 connection test failed, retrying... ($i/10)"
sleep 2
done
cd ../test/s3/copying
go test -v
kill -9 $pid || true
# Clean up data directory
rm -rf "$WEED_DATA_DIR" || true
sql-store-tests:
name: Basic S3 tests (SQL store)
runs-on: ubuntu-22.04
timeout-minutes: 15
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
- name: Set up Go 1.x
uses: actions/setup-go@v5.5.0
with:
go-version-file: 'go.mod'
id: go
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.9'
- name: Clone s3-tests
run: |
git clone https://github.com/ceph/s3-tests.git
cd s3-tests
pip install -r requirements.txt
pip install tox
pip install -e .
- name: Run Ceph S3 tests with SQL store
timeout-minutes: 15
env:
S3TEST_CONF: ../docker/compose/s3tests.conf
shell: bash
run: |
cd weed
# Debug: Check for port conflicts before starting
echo "=== Pre-start Port Check ==="
netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" || echo "Ports are free"
# Kill any existing weed processes that might interfere
echo "=== Cleanup existing processes ==="
pkill -f weed || echo "No weed processes found"
# More aggressive port cleanup using multiple methods
for port in 9337 8085 8892 8004 9328; do
echo "Cleaning port $port..."
# Method 1: lsof
pid=$(lsof -ti :$port 2>/dev/null || echo "")
if [ -n "$pid" ]; then
echo "Found process $pid using port $port (via lsof)"
kill -9 $pid 2>/dev/null || echo "Failed to kill $pid"
fi
# Method 2: netstat + ps (for cases where lsof fails)
netstat_pids=$(netstat -tlnp 2>/dev/null | grep ":$port " | awk '{print $7}' | cut -d'/' -f1 | grep -v '^-$' || echo "")
for npid in $netstat_pids; do
if [ -n "$npid" ] && [ "$npid" != "-" ]; then
echo "Found process $npid using port $port (via netstat)"
kill -9 $npid 2>/dev/null || echo "Failed to kill $npid"
fi
done
# Method 3: fuser (if available)
if command -v fuser >/dev/null 2>&1; then
fuser -k ${port}/tcp 2>/dev/null || echo "No process found via fuser for port $port"
fi
sleep 1
done
# Wait for ports to be released
sleep 5
echo "=== Post-cleanup Port Check ==="
netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" || echo "All ports are now free"
# If any ports are still in use, fail fast
if netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" >/dev/null 2>&1; then
echo "❌ ERROR: Some ports are still in use after aggressive cleanup!"
echo "=== Detailed Port Analysis ==="
for port in 9337 8085 8892 8004 9328; do
echo "Port $port:"
netstat -tlnp 2>/dev/null | grep ":$port " || echo " Not in use"
lsof -i :$port 2>/dev/null || echo " No lsof info"
done
exit 1
fi
go install -tags "sqlite" -buildvcs=false
# Create clean data directory for this test run with unique timestamp and process ID
export WEED_DATA_DIR="/tmp/seaweedfs-sql-test-$(date +%s)-$$"
# Create clean data directory for this test run
export WEED_DATA_DIR="/tmp/seaweedfs-sql-test-$(date +%s)"
mkdir -p "$WEED_DATA_DIR"
chmod 777 "$WEED_DATA_DIR"
# SQLite-specific configuration
export WEED_LEVELDB2_ENABLED="false"
export WEED_SQLITE_ENABLED="true"
export WEED_SQLITE_DBFILE="$WEED_DATA_DIR/filer.db"
echo "=== SQL Store Configuration ==="
echo "Data Dir: $WEED_DATA_DIR"
echo "SQLite DB: $WEED_SQLITE_DBFILE"
echo "LEVELDB2_ENABLED: $WEED_LEVELDB2_ENABLED"
echo "SQLITE_ENABLED: $WEED_SQLITE_ENABLED"
export WEED_LEVELDB2_ENABLED="false" WEED_SQLITE_ENABLED="true" WEED_SQLITE_DBFILE="$WEED_DATA_DIR/filer.db"
set -x
weed -v 1 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
-dir="$WEED_DATA_DIR" \
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
-volume.max=100 -volume.preStopSeconds=1 \
-master.port=9337 -volume.port=8085 -filer.port=8892 -s3.port=8004 -metricsPort=9328 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json \
> /tmp/seaweedfs-sql-server.log 2>&1 &
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
pid=$!
echo "=== Server started with PID: $pid ==="
# Wait for all SeaweedFS components to be ready
echo "Waiting for SeaweedFS components to start..."
# Check if server process is still alive before waiting
if ! kill -0 $pid 2>/dev/null; then
echo "❌ Server process died immediately after start"
echo "=== Immediate Log Check ==="
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null || echo "No log available"
exit 1
fi
sleep 5 # Give SQLite more time to initialize
for i in {1..30}; do
if curl -s http://localhost:9337/cluster/status > /dev/null 2>&1; then
echo "Master server is ready"
break
fi
echo "Waiting for master server... ($i/30)"
# Check if server process is still alive
if ! kill -0 $pid 2>/dev/null; then
echo "❌ Server process died while waiting for master"
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
exit 1
fi
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8085/status > /dev/null 2>&1; then
echo "Volume server is ready"
break
fi
echo "Waiting for volume server... ($i/30)"
if ! kill -0 $pid 2>/dev/null; then
echo "❌ Server process died while waiting for volume"
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
exit 1
fi
sleep 2
done
for i in {1..30}; do
if curl -s http://localhost:8892/ > /dev/null 2>&1; then
echo "Filer (SQLite) is ready"
break
fi
echo "Waiting for filer (SQLite)... ($i/30)"
if ! kill -0 $pid 2>/dev/null; then
echo "❌ Server process died while waiting for filer"
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
exit 1
fi
sleep 2
done
# Extra wait for SQLite filer to fully initialize
echo "Giving SQLite filer extra time to initialize..."
sleep 5
for i in {1..30}; do
if curl -s http://localhost:8004/ > /dev/null 2>&1; then
echo "S3 server is ready"
break
fi
echo "Waiting for S3 server... ($i/30)"
if ! kill -0 $pid 2>/dev/null; then
echo "❌ Server process died while waiting for S3"
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
exit 1
fi
sleep 2
done
echo "All SeaweedFS components are ready!"
sleep 10
cd ../s3-tests
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
# Create and update s3tests.conf to use port 8004
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-sql.conf
sed -i 's/port = 8000/port = 8004/g' ../docker/compose/s3tests-sql.conf
sed -i 's/:8000/:8004/g' ../docker/compose/s3tests-sql.conf
sed -i 's/localhost:8000/localhost:8004/g' ../docker/compose/s3tests-sql.conf
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8004/g' ../docker/compose/s3tests-sql.conf
export S3TEST_CONF=../docker/compose/s3tests-sql.conf
# Debug: Show the config file contents
echo "=== S3 Config File Contents ==="
cat ../docker/compose/s3tests-sql.conf
echo "=== End Config ==="
# Additional wait for S3-Filer integration to be fully ready
echo "Waiting additional 10 seconds for S3-Filer integration..."
sleep 10
# Test S3 connection before running tests
echo "Testing S3 connection..."
# Debug: Check if SeaweedFS processes are running
echo "=== Process Status ==="
ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
# Debug: Check port status
echo "=== Port Status ==="
netstat -tulpn | grep -E "(8004|9337|8085|8892)" || echo "Ports not found"
# Debug: Check server logs
echo "=== Recent Server Logs ==="
echo "--- SQL Server Log ---"
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null || echo "No SQL server log found"
echo "--- Other Logs ---"
ls -la /tmp/seaweedfs-*.log 2>/dev/null || echo "No other log files found"
for i in {1..10}; do
if curl -s -f http://localhost:8004/ > /dev/null 2>&1; then
echo "S3 connection test successful"
break
fi
echo "S3 connection test failed, retrying... ($i/10)"
# Debug: Try different HTTP methods
echo "Debug: Testing different endpoints..."
curl -s -I http://localhost:8004/ || echo "HEAD request failed"
curl -s http://localhost:8004/status || echo "Status endpoint failed"
sleep 2
done
tox -- \
s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
-2
View File
@@ -110,5 +110,3 @@ test/s3/cors/cors.test
/test/s3/retention/filerldb2
test/s3/retention/weed-server.pid
test/s3/retention/weed-test.log
/test/s3/versioning/test-volume-data
test/s3/versioning/weed-test.log
+1 -1
View File
@@ -23,7 +23,7 @@ server: install
benchmark: install warp_install
pkill weed || true
pkill warp || true
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
warp client &
while ! nc -z localhost 8000 ; do sleep 1 ; done
warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm
+2 -2
View File
@@ -10,7 +10,7 @@ services:
- 18084:18080
- 8888:8888
- 18888:18888
command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
volumes:
- ./master-cloud.toml:/etc/seaweedfs/master.toml
depends_on:
@@ -25,4 +25,4 @@ services:
- 8889:8888
- 18889:18888
- 8334:8333
command: "server -ip=server2 -filer -s3 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
command: "server -ip=server2 -filer -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
@@ -3,7 +3,7 @@ version: '3.9'
services:
server-left:
image: chrislusf/seaweedfs:local
command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
healthcheck:
@@ -13,7 +13,7 @@ services:
timeout: 30s
server-right:
image: chrislusf/seaweedfs:local
command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
volumes:
- ./s3.json:/etc/seaweedfs/s3.json
healthcheck:
@@ -6,7 +6,7 @@ services:
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -volumeSizeLimitMB=100"
command: "master -ip=master -volumeSizeLimitMB=1024"
volume:
image: chrislusf/seaweedfs:local
ports:
+1 -1
View File
@@ -6,7 +6,7 @@ services:
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -volumeSizeLimitMB=100"
command: "master -ip=master -volumeSizeLimitMB=1024"
volume:
image: chrislusf/seaweedfs:local
ports:
+1 -1
View File
@@ -11,7 +11,7 @@ services:
ports:
- 9333:9333
- 19333:19333
command: "master -ip=master -volumeSizeLimitMB=100"
command: "master -ip=master -volumeSizeLimitMB=1024"
volume:
image: chrislusf/seaweedfs:local
ports:
+2 -2
View File
@@ -1,6 +1,6 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "3.95"
appVersion: "3.94"
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
version: 4.0.395
version: 4.0.394
+2 -2
View File
@@ -222,13 +222,13 @@ test-with-server: start-server
test-versioning-with-configs: check-deps
@echo "Testing with different S3 configurations..."
@echo "Testing with empty folder allowed..."
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowEmptyFolder=true -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowEmptyFolder=true -filer -master.volumeSizeLimitMB=1024 -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
@sleep 5
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
@if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi
@sleep 2
@echo "Testing with delete bucket not empty disabled..."
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=1024 -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
@sleep 5
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
@if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi
File diff suppressed because it is too large Load Diff
+2 -13
View File
@@ -164,16 +164,6 @@ func checkVersioningStatus(t *testing.T, client *s3.Client, bucketName string, e
assert.Equal(t, expectedStatus, resp.Status)
}
// checkVersioningStatusEmpty verifies that a bucket has no versioning configuration (newly created bucket)
func checkVersioningStatusEmpty(t *testing.T, client *s3.Client, bucketName string) {
resp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
Bucket: aws.String(bucketName),
})
require.NoError(t, err)
// AWS S3 returns an empty versioning configuration (no Status field) for buckets that have never had versioning configured, such as newly created buckets.
assert.Empty(t, resp.Status, "Newly created bucket should have empty versioning status")
}
// putObject puts an object into a bucket
func putObject(t *testing.T, client *s3.Client, bucketName, key, content string) *s3.PutObjectOutput {
resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
@@ -294,9 +284,8 @@ func TestVersioningBasicWorkflow(t *testing.T) {
createBucket(t, client, bucketName)
defer deleteBucket(t, client, bucketName)
// Initially, versioning should be unset/empty (not suspended) for newly created buckets
// This matches AWS S3 behavior where new buckets have no versioning status
checkVersioningStatusEmpty(t, client, bucketName)
// Initially, versioning should be suspended/disabled
checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusSuspended)
// Enable versioning
enableVersioning(t, client, bucketName)
Binary file not shown.
+1 -1
View File
@@ -23,7 +23,7 @@ debug_mount:
debug_server:
go build -gcflags="all=-N -l"
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
debug_volume:
go build -tags=5BytesOffset -gcflags="all=-N -l"
+3 -2
View File
@@ -138,8 +138,9 @@ func (r *LockRing) compactSnapshots() {
r.Lock()
defer r.Unlock()
// Always attempt compaction when called, regardless of lastCompactTime
// This ensures proper cleanup even with multiple concurrent compaction requests
if r.lastCompactTime.After(r.lastUpdateTime) {
return
}
ts := time.Now()
// remove old snapshots
+39 -91
View File
@@ -51,13 +51,6 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
entry.Extended = make(map[string][]byte)
}
entry.Extended["key"] = []byte(*input.Key)
// Set object owner for multipart upload
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
if amzAccountId != "" {
entry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(amzAccountId)
}
for k, v := range input.Metadata {
entry.Extended[k] = []byte(*v)
}
@@ -99,7 +92,7 @@ type CompleteMultipartUploadResult struct {
VersionId *string `xml:"-"`
}
func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) {
func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) {
glog.V(2).Infof("completeMultipartUpload input %v", input)
if len(parts.Parts) == 0 {
@@ -245,10 +238,32 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
}
entryName, dirName := s3a.getEntryNameAndDir(input)
err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) {
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.SeaweedFSUploadId] = []byte(*input.UploadId)
for k, v := range pentry.Extended {
if k != "key" {
entry.Extended[k] = v
}
}
if pentry.Attributes.Mime != "" {
entry.Attributes.Mime = pentry.Attributes.Mime
} else if mime != "" {
entry.Attributes.Mime = mime
}
entry.Attributes.FileSize = uint64(offset)
})
// Check if versioning is configured for this bucket BEFORE creating any files
versioningState, vErr := s3a.getVersioningState(*input.Bucket)
if vErr == nil && versioningState == s3_constants.VersioningEnabled {
if err != nil {
glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err)
return nil, s3err.ErrInternalError
}
// Check if versioning is enabled for this bucket
versioningEnabled, vErr := s3a.isVersioningEnabled(*input.Bucket)
if vErr == nil && versioningEnabled {
// For versioned buckets, create a version and return the version ID
versionId := generateVersionId()
versionFileName := s3a.getVersionFileName(versionId)
@@ -261,13 +276,6 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
}
versionEntry.Extended[s3_constants.ExtVersionIdKey] = []byte(versionId)
versionEntry.Extended[s3_constants.SeaweedFSUploadId] = []byte(*input.UploadId)
// Set object owner for versioned multipart objects
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
if amzAccountId != "" {
versionEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(amzAccountId)
}
for k, v := range pentry.Extended {
if k != "key" {
versionEntry.Extended[k] = v
@@ -293,8 +301,19 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
return nil, s3err.ErrInternalError
}
// For versioned buckets, don't create a main object file - all content is stored in .versions directory
// The latest version information is tracked in the .versions directory metadata
// Create a delete marker for the main object (latest version)
err = s3a.mkFile(dirName, entryName, nil, func(mainEntry *filer_pb.Entry) {
if mainEntry.Extended == nil {
mainEntry.Extended = make(map[string][]byte)
}
mainEntry.Extended[s3_constants.ExtVersionIdKey] = []byte(versionId)
mainEntry.Extended[s3_constants.ExtDeleteMarkerKey] = []byte("false") // This is the latest version, not a delete marker
})
if err != nil {
glog.Errorf("completeMultipartUpload: failed to update main entry: %v", err)
return nil, s3err.ErrInternalError
}
output = &CompleteMultipartUploadResult{
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))),
@@ -303,78 +322,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
Key: objectKey(input.Key),
VersionId: aws.String(versionId),
}
} else if vErr == nil && versioningState == s3_constants.VersioningSuspended {
// For suspended versioning, add "null" version ID metadata and return "null" version ID
err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) {
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.ExtVersionIdKey] = []byte("null")
// Set object owner for suspended versioning multipart objects
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
if amzAccountId != "" {
entry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(amzAccountId)
}
for k, v := range pentry.Extended {
if k != "key" {
entry.Extended[k] = v
}
}
if pentry.Attributes.Mime != "" {
entry.Attributes.Mime = pentry.Attributes.Mime
} else if mime != "" {
entry.Attributes.Mime = mime
}
entry.Attributes.FileSize = uint64(offset)
})
if err != nil {
glog.Errorf("completeMultipartUpload: failed to create suspended versioning object: %v", err)
return nil, s3err.ErrInternalError
}
// Note: Suspended versioning should NOT return VersionId field according to AWS S3 spec
output = &CompleteMultipartUploadResult{
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))),
Bucket: input.Bucket,
ETag: aws.String("\"" + filer.ETagChunks(finalParts) + "\""),
Key: objectKey(input.Key),
// VersionId field intentionally omitted for suspended versioning
}
} else {
// For non-versioned buckets, create main object file
err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) {
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.SeaweedFSUploadId] = []byte(*input.UploadId)
// Set object owner for non-versioned multipart objects
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
if amzAccountId != "" {
entry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(amzAccountId)
}
for k, v := range pentry.Extended {
if k != "key" {
entry.Extended[k] = v
}
}
if pentry.Attributes.Mime != "" {
entry.Attributes.Mime = pentry.Attributes.Mime
} else if mime != "" {
entry.Attributes.Mime = mime
}
entry.Attributes.FileSize = uint64(offset)
})
if err != nil {
glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err)
return nil, s3err.ErrInternalError
}
// For non-versioned buckets, return response without VersionId
output = &CompleteMultipartUploadResult{
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))),
+16 -50
View File
@@ -2,7 +2,6 @@ package s3api
import (
"encoding/json"
"errors"
"fmt"
"path/filepath"
"strings"
@@ -98,11 +97,10 @@ func (s3a *S3ApiServer) getBucketConfig(bucket string) (*BucketConfig, s3err.Err
return config, s3err.ErrNone
}
// Try to get from filer
entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
// Load from filer
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
// Bucket doesn't exist
if err == filer_pb.ErrNotFound {
return nil, s3err.ErrNoSuchBucket
}
glog.Errorf("getBucketConfig: failed to get bucket entry for %s: %v", bucket, err)
@@ -111,25 +109,25 @@ func (s3a *S3ApiServer) getBucketConfig(bucket string) (*BucketConfig, s3err.Err
config := &BucketConfig{
Name: bucket,
Entry: entry,
Entry: bucketEntry,
}
// Extract configuration from extended attributes
if entry.Extended != nil {
if versioning, exists := entry.Extended[s3_constants.ExtVersioningKey]; exists {
if bucketEntry.Extended != nil {
if versioning, exists := bucketEntry.Extended[s3_constants.ExtVersioningKey]; exists {
config.Versioning = string(versioning)
}
if ownership, exists := entry.Extended[s3_constants.ExtOwnershipKey]; exists {
if ownership, exists := bucketEntry.Extended[s3_constants.ExtOwnershipKey]; exists {
config.Ownership = string(ownership)
}
if acl, exists := entry.Extended[s3_constants.ExtAmzAclKey]; exists {
if acl, exists := bucketEntry.Extended[s3_constants.ExtAmzAclKey]; exists {
config.ACL = acl
}
if owner, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
if owner, exists := bucketEntry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
config.Owner = string(owner)
}
// Parse Object Lock configuration if present
if objectLockConfig, found := LoadObjectLockConfigurationFromExtended(entry); found {
if objectLockConfig, found := LoadObjectLockConfigurationFromExtended(bucketEntry); found {
config.ObjectLockConfig = objectLockConfig
glog.V(2).Infof("getBucketConfig: cached Object Lock configuration for bucket %s", bucket)
}
@@ -137,7 +135,7 @@ func (s3a *S3ApiServer) getBucketConfig(bucket string) (*BucketConfig, s3err.Err
// Load CORS configuration from .s3metadata
if corsConfig, err := s3a.loadCORSFromMetadata(bucket); err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
if err == filer_pb.ErrNotFound {
// Missing metadata is not an error; fall back cleanly
glog.V(2).Infof("CORS metadata not found for bucket %s, falling back to default behavior", bucket)
} else {
@@ -221,40 +219,6 @@ func (s3a *S3ApiServer) isVersioningEnabled(bucket string) (bool, error) {
return config.Versioning == s3_constants.VersioningEnabled || config.ObjectLockConfig != nil, nil
}
// isVersioningConfigured checks if versioning has been configured (either Enabled or Suspended)
func (s3a *S3ApiServer) isVersioningConfigured(bucket string) (bool, error) {
config, errCode := s3a.getBucketConfig(bucket)
if errCode != s3err.ErrNone {
if errCode == s3err.ErrNoSuchBucket {
return false, filer_pb.ErrNotFound
}
return false, fmt.Errorf("failed to get bucket config: %v", errCode)
}
// Versioning is configured if explicitly set to either "Enabled" or "Suspended"
// OR if object lock is enabled (which forces versioning)
return config.Versioning != "" || config.ObjectLockConfig != nil, nil
}
// getVersioningState returns the detailed versioning state for a bucket
func (s3a *S3ApiServer) getVersioningState(bucket string) (string, error) {
config, errCode := s3a.getBucketConfig(bucket)
if errCode != s3err.ErrNone {
if errCode == s3err.ErrNoSuchBucket {
return "", filer_pb.ErrNotFound
}
return "", fmt.Errorf("failed to get bucket config: %v", errCode)
}
// If object lock is enabled, versioning must be enabled regardless of explicit setting
if config.ObjectLockConfig != nil {
return s3_constants.VersioningEnabled, nil
}
// Return the explicit versioning status (empty string means never configured)
return config.Versioning, nil
}
// getBucketVersioningStatus returns the versioning status for a bucket
func (s3a *S3ApiServer) getBucketVersioningStatus(bucket string) (string, s3err.ErrorCode) {
config, errCode := s3a.getBucketConfig(bucket)
@@ -262,8 +226,10 @@ func (s3a *S3ApiServer) getBucketVersioningStatus(bucket string) (string, s3err.
return "", errCode
}
// Return exactly what's stored - empty string means versioning was never configured
// This matches AWS S3 behavior where new buckets have no Status field in GetBucketVersioning response
if config.Versioning == "" {
return s3_constants.VersioningSuspended, s3err.ErrNone
}
return config.Versioning, s3err.ErrNone
}
@@ -312,7 +278,7 @@ func (s3a *S3ApiServer) loadCORSFromMetadata(bucket string) (*cors.CORSConfigura
entry, err := s3a.getEntry("", bucketMetadataPath)
if err != nil {
glog.V(3).Infof("loadCORSFromMetadata: error retrieving metadata for bucket %s: %v", bucket, err)
return nil, fmt.Errorf("error retrieving CORS metadata for bucket %s: %w", bucket, err)
return nil, fmt.Errorf("error retrieving metadata for bucket %s: %v", bucket, err)
}
if entry == nil {
glog.V(3).Infof("loadCORSFromMetadata: no metadata entry found for bucket %s", bucket)
+8 -19
View File
@@ -230,7 +230,7 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("HeadBucketHandler %s", bucket)
if entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket); entry == nil || errors.Is(err, filer_pb.ErrNotFound) {
if entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket); entry == nil || err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
@@ -240,7 +240,7 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorCode {
entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
if entry == nil || errors.Is(err, filer_pb.ErrNotFound) {
if entry == nil || err == filer_pb.ErrNotFound {
return s3err.ErrNoSuchBucket
}
@@ -669,7 +669,7 @@ func (s3a *S3ApiServer) DeleteBucketOwnershipControls(w http.ResponseWriter, r *
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
@@ -714,22 +714,11 @@ func (s3a *S3ApiServer) GetBucketVersioningHandler(w http.ResponseWriter, r *htt
return
}
// AWS S3 behavior: If versioning was never configured, don't return Status field
var response *s3.PutBucketVersioningInput
if versioningStatus == "" {
// No versioning configuration - return empty response (no Status field)
response = &s3.PutBucketVersioningInput{
VersioningConfiguration: &s3.VersioningConfiguration{},
}
} else {
// Versioning was explicitly configured - return the status
response = &s3.PutBucketVersioningInput{
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String(versioningStatus),
},
}
}
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, response)
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, &s3.PutBucketVersioningInput{
VersioningConfiguration: &s3.VersioningConfiguration{
Status: aws.String(versioningStatus),
},
})
}
// PutBucketVersioningHandler Put bucket Versioning
+24 -165
View File
@@ -2,7 +2,6 @@ package s3api
import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
@@ -86,95 +85,6 @@ func removeDuplicateSlashes(object string) string {
return result.String()
}
// checkDirectoryObject checks if the object is a directory object (ends with "/") and if it exists
// Returns: (entry, isDirectoryObject, error)
// - entry: the directory entry if found and is a directory
// - isDirectoryObject: true if the request was for a directory object (ends with "/")
// - error: any error encountered while checking
func (s3a *S3ApiServer) checkDirectoryObject(bucket, object string) (*filer_pb.Entry, bool, error) {
if !strings.HasSuffix(object, "/") {
return nil, false, nil // Not a directory object
}
bucketDir := s3a.option.BucketsPath + "/" + bucket
cleanObject := strings.TrimSuffix(strings.TrimPrefix(object, "/"), "/")
if cleanObject == "" {
return nil, true, nil // Root level directory object, but we don't handle it
}
// Check if directory exists
dirEntry, err := s3a.getEntry(bucketDir, cleanObject)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
return nil, true, nil // Directory object requested but doesn't exist
}
return nil, true, err // Other errors should be propagated
}
if !dirEntry.IsDirectory {
return nil, true, nil // Exists but not a directory
}
return dirEntry, true, nil
}
// serveDirectoryContent serves the content of a directory object directly
func (s3a *S3ApiServer) serveDirectoryContent(w http.ResponseWriter, r *http.Request, entry *filer_pb.Entry) {
// Set content type - use stored MIME type or default
contentType := entry.Attributes.Mime
if contentType == "" {
contentType = "application/octet-stream"
}
w.Header().Set("Content-Type", contentType)
// Set content length - use FileSize for accuracy, especially for large files
contentLength := int64(entry.Attributes.FileSize)
w.Header().Set("Content-Length", strconv.FormatInt(contentLength, 10))
// Set last modified
w.Header().Set("Last-Modified", time.Unix(entry.Attributes.Mtime, 0).UTC().Format(http.TimeFormat))
// Set ETag
w.Header().Set("ETag", "\""+filer.ETag(entry)+"\"")
// For HEAD requests, don't write body
if r.Method == http.MethodHead {
w.WriteHeader(http.StatusOK)
return
}
// Write content
w.WriteHeader(http.StatusOK)
if len(entry.Content) > 0 {
if _, err := w.Write(entry.Content); err != nil {
glog.Errorf("serveDirectoryContent: failed to write response: %v", err)
}
}
}
// handleDirectoryObjectRequest is a helper function that handles directory object requests
// for both GET and HEAD operations, eliminating code duplication
func (s3a *S3ApiServer) handleDirectoryObjectRequest(w http.ResponseWriter, r *http.Request, bucket, object, handlerName string) bool {
// Check if this is a directory object and handle it directly
if dirEntry, isDirectoryObject, err := s3a.checkDirectoryObject(bucket, object); err != nil {
glog.Errorf("%s: error checking directory object %s/%s: %v", handlerName, bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return true // Request was handled (with error)
} else if dirEntry != nil {
glog.V(2).Infof("%s: directory object %s/%s found, serving content", handlerName, bucket, object)
s3a.serveDirectoryContent(w, r, dirEntry)
return true // Request was handled successfully
} else if isDirectoryObject {
// Directory object but doesn't exist
glog.V(2).Infof("%s: directory object %s/%s not found", handlerName, bucket, object)
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return true // Request was handled (with not found)
}
return false // Not a directory object, continue with normal processing
}
func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bucketPrefix string, fetchOwner bool, isDirectory bool, encodingTypeUrl bool) (listEntry ListEntry) {
storageClass := "STANDARD"
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
@@ -198,31 +108,9 @@ func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bu
StorageClass: StorageClass(storageClass),
}
if fetchOwner {
// Extract owner from S3 metadata (Extended attributes) instead of file system attributes
var ownerID, displayName string
if entry.Extended != nil {
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
ownerID = string(ownerBytes)
}
}
// Fallback to anonymous if no S3 owner found
if ownerID == "" {
ownerID = s3_constants.AccountAnonymousId
displayName = "anonymous"
} else {
// Use the ownerID as displayName if no better option is available
displayName = ownerID
// Additional fallback to file system username if available and different from ownerID
if entry.Attributes.UserName != "" && entry.Attributes.UserName != ownerID {
displayName = entry.Attributes.UserName
}
}
listEntry.Owner = CanonicalUser{
ID: ownerID,
DisplayName: displayName,
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
DisplayName: entry.Attributes.UserName,
}
}
return listEntry
@@ -240,16 +128,16 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
bucket, object := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("GetObjectHandler %s %s", bucket, object)
// Handle directory objects with shared logic
if s3a.handleDirectoryObjectRequest(w, r, bucket, object, "GetObjectHandler") {
return // Directory object request was handled
if strings.HasSuffix(r.URL.Path, "/") {
s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
return
}
// Check for specific version ID in query parameters
versionId := r.URL.Query().Get("versionId")
// Check if versioning is configured for the bucket (Enabled or Suspended)
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
// Check if versioning is enabled for the bucket
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
@@ -260,18 +148,16 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
return
}
glog.V(1).Infof("GetObject: bucket %s, object %s, versioningConfigured=%v, versionId=%s", bucket, object, versioningConfigured, versionId)
var destUrl string
if versioningConfigured {
if versioningEnabled {
// Handle versioned GET - all versions are stored in .versions directory
var targetVersionId string
var entry *filer_pb.Entry
if versionId != "" {
// Request for specific version
glog.V(2).Infof("GetObject: requesting specific version %s for %s%s", versionId, bucket, object)
glog.V(2).Infof("GetObject: requesting specific version %s for %s/%s", versionId, bucket, object)
entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
if err != nil {
glog.Errorf("Failed to get specific version %s: %v", versionId, err)
@@ -281,10 +167,10 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
targetVersionId = versionId
} else {
// Request for latest version
glog.V(1).Infof("GetObject: requesting latest version for %s%s", bucket, object)
glog.V(2).Infof("GetObject: requesting latest version for %s/%s", bucket, object)
entry, err = s3a.getLatestObjectVersion(bucket, object)
if err != nil {
glog.Errorf("GetObject: Failed to get latest version for %s%s: %v", bucket, object, err)
glog.Errorf("Failed to get latest version: %v", err)
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return
}
@@ -293,10 +179,6 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
targetVersionId = string(versionIdBytes)
}
}
// If no version ID found in entry, this is a pre-versioning object
if targetVersionId == "" {
targetVersionId = "null"
}
}
// Check if this is a delete marker
@@ -307,17 +189,10 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
}
}
// Determine the actual file path based on whether this is a versioned or pre-versioning object
if targetVersionId == "null" {
// Pre-versioning object - stored as regular file
destUrl = s3a.toFilerUrl(bucket, object)
glog.V(2).Infof("GetObject: pre-versioning object URL: %s", destUrl)
} else {
// Versioned object - stored in .versions directory
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
glog.V(2).Infof("GetObject: version %s URL: %s", targetVersionId, destUrl)
}
// All versions are stored in .versions directory
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
glog.V(2).Infof("GetObject: version %s URL: %s", targetVersionId, destUrl)
// Set version ID in response header
w.Header().Set("x-amz-version-id", targetVersionId)
@@ -337,16 +212,11 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
bucket, object := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("HeadObjectHandler %s %s", bucket, object)
// Handle directory objects with shared logic
if s3a.handleDirectoryObjectRequest(w, r, bucket, object, "HeadObjectHandler") {
return // Directory object request was handled
}
// Check for specific version ID in query parameters
versionId := r.URL.Query().Get("versionId")
// Check if versioning is configured for the bucket (Enabled or Suspended)
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
// Check if versioning is enabled for the bucket
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
@@ -359,14 +229,14 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
var destUrl string
if versioningConfigured {
if versioningEnabled {
// Handle versioned HEAD - all versions are stored in .versions directory
var targetVersionId string
var entry *filer_pb.Entry
if versionId != "" {
// Request for specific version
glog.V(2).Infof("HeadObject: requesting specific version %s for %s%s", versionId, bucket, object)
glog.V(2).Infof("HeadObject: requesting specific version %s for %s/%s", versionId, bucket, object)
entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
if err != nil {
glog.Errorf("Failed to get specific version %s: %v", versionId, err)
@@ -376,7 +246,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
targetVersionId = versionId
} else {
// Request for latest version
glog.V(2).Infof("HeadObject: requesting latest version for %s%s", bucket, object)
glog.V(2).Infof("HeadObject: requesting latest version for %s/%s", bucket, object)
entry, err = s3a.getLatestObjectVersion(bucket, object)
if err != nil {
glog.Errorf("Failed to get latest version: %v", err)
@@ -388,10 +258,6 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
targetVersionId = string(versionIdBytes)
}
}
// If no version ID found in entry, this is a pre-versioning object
if targetVersionId == "" {
targetVersionId = "null"
}
}
// Check if this is a delete marker
@@ -402,17 +268,10 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
}
}
// Determine the actual file path based on whether this is a versioned or pre-versioning object
if targetVersionId == "null" {
// Pre-versioning object - stored as regular file
destUrl = s3a.toFilerUrl(bucket, object)
glog.V(2).Infof("HeadObject: pre-versioning object URL: %s", destUrl)
} else {
// Versioned object - stored in .versions directory
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
glog.V(2).Infof("HeadObject: version %s URL: %s", targetVersionId, destUrl)
}
// All versions are stored in .versions directory
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
glog.V(2).Infof("HeadObject: version %s URL: %s", targetVersionId, destUrl)
// Set version ID in response header
w.Header().Set("x-amz-version-id", targetVersionId)
-356
View File
@@ -1,356 +0,0 @@
package s3api
import (
"context"
"errors"
"fmt"
"net/http"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
// GetObjectAclHandler Get object ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html
func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Request) {
// collect parameters
bucket, object := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("GetObjectAclHandler %s %s", bucket, object)
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, err)
return
}
// Check for specific version ID in query parameters
versionId := r.URL.Query().Get("versionId")
// Check if versioning is configured for the bucket (Enabled or Suspended)
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
glog.Errorf("GetObjectAclHandler: Error checking versioning status for bucket %s: %v", bucket, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
var entry *filer_pb.Entry
if versioningConfigured {
// Handle versioned object ACL retrieval - use same logic as GetObjectHandler
if versionId != "" {
// Request for specific version
glog.V(2).Infof("GetObjectAclHandler: requesting ACL for specific version %s of %s%s", versionId, bucket, object)
entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
} else {
// Request for latest version
glog.V(2).Infof("GetObjectAclHandler: requesting ACL for latest version of %s%s", bucket, object)
entry, err = s3a.getLatestObjectVersion(bucket, object)
}
if err != nil {
glog.Errorf("GetObjectAclHandler: Failed to get object version %s for %s%s: %v", versionId, bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return
}
// Check if this is a delete marker
if entry.Extended != nil {
if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return
}
}
} else {
// Handle regular (non-versioned) object ACL retrieval
bucketDir := s3a.option.BucketsPath + "/" + bucket
entry, err = s3a.getEntry(bucketDir, object)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return
}
glog.Errorf("GetObjectAclHandler: error checking object %s/%s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
}
if entry == nil {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return
}
// Get object owner from metadata, fallback to request account
var objectOwner string
var objectOwnerDisplayName string
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
if entry.Extended != nil {
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
objectOwner = string(ownerBytes)
}
}
// Fallback to current account if no owner stored
if objectOwner == "" {
objectOwner = amzAccountId
}
objectOwnerDisplayName = s3a.iam.GetAccountNameById(objectOwner)
// Build ACL response
response := AccessControlPolicy{
Owner: CanonicalUser{
ID: objectOwner,
DisplayName: objectOwnerDisplayName,
},
}
// Get grants from stored ACL metadata
grants := GetAcpGrants(entry.Extended)
if len(grants) > 0 {
// Convert AWS SDK grants to local Grant format
for _, grant := range grants {
localGrant := Grant{
Permission: Permission(*grant.Permission),
}
if grant.Grantee != nil {
localGrant.Grantee = Grantee{
Type: *grant.Grantee.Type,
XMLXSI: "CanonicalUser",
XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
}
if grant.Grantee.ID != nil {
localGrant.Grantee.ID = *grant.Grantee.ID
localGrant.Grantee.DisplayName = s3a.iam.GetAccountNameById(*grant.Grantee.ID)
}
if grant.Grantee.URI != nil {
localGrant.Grantee.URI = *grant.Grantee.URI
}
}
response.AccessControlList.Grant = append(response.AccessControlList.Grant, localGrant)
}
} else {
// Fallback to default full control for object owner
response.AccessControlList.Grant = append(response.AccessControlList.Grant, Grant{
Grantee: Grantee{
ID: objectOwner,
DisplayName: objectOwnerDisplayName,
Type: "CanonicalUser",
XMLXSI: "CanonicalUser",
XMLNS: "http://www.w3.org/2001/XMLSchema-instance"},
Permission: Permission(s3_constants.PermissionFullControl),
})
}
writeSuccessResponseXML(w, r, response)
}
// PutObjectAclHandler Put object ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html
func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) {
// collect parameters
bucket, object := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("PutObjectAclHandler %s %s", bucket, object)
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, err)
return
}
// Check for specific version ID in query parameters
versionId := r.URL.Query().Get("versionId")
// Check if versioning is configured for the bucket (Enabled or Suspended)
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
glog.Errorf("PutObjectAclHandler: Error checking versioning status for bucket %s: %v", bucket, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
var entry *filer_pb.Entry
if versioningConfigured {
// Handle versioned object ACL modification - use same logic as GetObjectHandler
if versionId != "" {
// Request for specific version
glog.V(2).Infof("PutObjectAclHandler: modifying ACL for specific version %s of %s%s", versionId, bucket, object)
entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
} else {
// Request for latest version
glog.V(2).Infof("PutObjectAclHandler: modifying ACL for latest version of %s%s", bucket, object)
entry, err = s3a.getLatestObjectVersion(bucket, object)
}
if err != nil {
glog.Errorf("PutObjectAclHandler: Failed to get object version %s for %s%s: %v", versionId, bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return
}
// Check if this is a delete marker
if entry.Extended != nil {
if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return
}
}
} else {
// Handle regular (non-versioned) object ACL modification
bucketDir := s3a.option.BucketsPath + "/" + bucket
entry, err = s3a.getEntry(bucketDir, object)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return
}
glog.Errorf("PutObjectAclHandler: error checking object %s/%s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
}
if entry == nil {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
return
}
// Get current object owner from metadata
var objectOwner string
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
if entry.Extended != nil {
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
objectOwner = string(ownerBytes)
}
}
// Fallback to current account if no owner stored
if objectOwner == "" {
objectOwner = amzAccountId
}
// **PERMISSION CHECKS**
// 1. Check if user is admin (admins can modify any ACL)
if !s3a.isUserAdmin(r) {
// 2. Check object ownership - only object owner can modify ACL (unless admin)
if objectOwner != amzAccountId {
glog.V(3).Infof("PutObjectAclHandler: Access denied - user %s is not owner of object %s/%s (owner: %s)",
amzAccountId, bucket, object, objectOwner)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
// 3. Check object-level WRITE_ACP permission
// Create the specific action for this object
writeAcpAction := Action(fmt.Sprintf("WriteAcp:%s/%s", bucket, object))
identity, errCode := s3a.iam.authRequest(r, writeAcpAction)
if errCode != s3err.ErrNone {
glog.V(3).Infof("PutObjectAclHandler: Auth failed for WriteAcp action on %s/%s: %v", bucket, object, errCode)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
// 4. Verify the authenticated identity can perform WriteAcp on this specific object
if identity == nil || !identity.canDo(writeAcpAction, bucket, object) {
glog.V(3).Infof("PutObjectAclHandler: Identity %v cannot perform WriteAcp on %s/%s", identity, bucket, object)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
} else {
glog.V(3).Infof("PutObjectAclHandler: Admin user %s granted ACL modification permission for %s/%s", amzAccountId, bucket, object)
}
// Get bucket config for ownership settings
bucketConfig, errCode := s3a.getBucketConfig(bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
bucketOwnership := bucketConfig.Ownership
bucketOwnerId := bucketConfig.Owner
// Extract ACL from request (either canned ACL or XML body)
// This function also validates that the owner in the request matches the object owner
grants, errCode := ExtractAcl(r, s3a.iam, bucketOwnership, bucketOwnerId, objectOwner, amzAccountId)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
// Store ACL in object metadata
if errCode := AssembleEntryWithAcp(entry, objectOwner, grants); errCode != s3err.ErrNone {
glog.Errorf("PutObjectAclHandler: failed to assemble entry with ACP: %v", errCode)
s3err.WriteErrorResponse(w, r, errCode)
return
}
// Calculate the correct directory for ACL update
var updateDirectory string
if versioningConfigured {
if versionId != "" && versionId != "null" {
// Versioned object - update the specific version file in .versions directory
updateDirectory = s3a.option.BucketsPath + "/" + bucket + "/" + object + ".versions"
} else {
// Latest version in versioned bucket - could be null version or versioned object
// Extract version ID from the entry to determine where it's stored
var actualVersionId string
if entry.Extended != nil {
if versionIdBytes, exists := entry.Extended[s3_constants.ExtVersionIdKey]; exists {
actualVersionId = string(versionIdBytes)
}
}
if actualVersionId == "null" || actualVersionId == "" {
// Null version (pre-versioning object) - stored as regular file
updateDirectory = s3a.option.BucketsPath + "/" + bucket
} else {
// Versioned object - stored in .versions directory
updateDirectory = s3a.option.BucketsPath + "/" + bucket + "/" + object + ".versions"
}
}
} else {
// Non-versioned object - stored as regular file
updateDirectory = s3a.option.BucketsPath + "/" + bucket
}
// Update the object with new ACL metadata
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.UpdateEntryRequest{
Directory: updateDirectory,
Entry: entry,
}
if _, err := client.UpdateEntry(context.Background(), request); err != nil {
return err
}
return nil
})
if err != nil {
glog.Errorf("PutObjectAclHandler: failed to update entry: %v", err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
glog.V(3).Infof("PutObjectAclHandler: Successfully updated ACL for %s/%s by user %s", bucket, object, amzAccountId)
writeSuccessResponseEmpty(w, r)
}
+39 -178
View File
@@ -38,9 +38,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
}
srcBucket, srcObject, srcVersionId := pathToBucketObjectAndVersion(cpSrcPath)
srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
glog.V(3).Infof("CopyObjectHandler %s %s (version: %s) => %s %s", srcBucket, srcObject, srcVersionId, dstBucket, dstObject)
glog.V(3).Infof("CopyObjectHandler %s %s => %s %s", srcBucket, srcObject, dstBucket, dstObject)
replaceMeta, replaceTagging := replaceDirective(r.Header)
@@ -76,41 +76,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
return
}
// Get detailed versioning state for source bucket
srcVersioningState, err := s3a.getVersioningState(srcBucket)
if err != nil {
glog.Errorf("Error checking versioning state for source bucket %s: %v", srcBucket, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
return
}
// Get the source entry with version awareness based on versioning state
var entry *filer_pb.Entry
if srcVersionId != "" {
// Specific version requested - always use version-aware retrieval
entry, err = s3a.getSpecificObjectVersion(srcBucket, srcObject, srcVersionId)
} else if srcVersioningState == s3_constants.VersioningEnabled {
// Versioning enabled - get latest version from .versions directory
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
} else if srcVersioningState == s3_constants.VersioningSuspended {
// Versioning suspended - current object is stored as regular file ("null" version)
// Try regular file first, fall back to latest version if needed
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err = s3a.getEntry(dir, name)
if err != nil {
// If regular file doesn't exist, try latest version as fallback
glog.V(2).Infof("CopyObject: regular file not found for suspended versioning, trying latest version")
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
}
} else {
// No versioning configured - use regular retrieval
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err = s3a.getEntry(dir, name)
}
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err := s3a.getEntry(dir, name)
if err != nil || entry.IsDirectory {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
return
@@ -170,108 +138,43 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
dstEntry.Chunks = dstChunks
}
// Check if destination bucket has versioning configured
dstVersioningConfigured, err := s3a.isVersioningConfigured(dstBucket)
if err != nil {
glog.Errorf("Error checking versioning status for destination bucket %s: %v", dstBucket, err)
// Save the new entry
dstPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
dstDir, dstName := dstPath.DirAndName()
// Check if destination exists and remove it first (S3 copy overwrites)
if exists, _ := s3a.exists(dstDir, dstName, false); exists {
if err := s3a.rm(dstDir, dstName, false, false); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
}
// Create the new file
if err := s3a.mkFile(dstDir, dstName, dstEntry.Chunks, func(entry *filer_pb.Entry) {
entry.Attributes = dstEntry.Attributes
entry.Extended = dstEntry.Extended
}); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
var dstVersionId string
var etag string
if dstVersioningConfigured {
// For versioned destination, create a new version
dstVersionId = generateVersionId()
glog.V(2).Infof("CopyObjectHandler: creating version %s for destination %s/%s", dstVersionId, dstBucket, dstObject)
// Add version metadata to the entry
if dstEntry.Extended == nil {
dstEntry.Extended = make(map[string][]byte)
}
dstEntry.Extended[s3_constants.ExtVersionIdKey] = []byte(dstVersionId)
// Calculate ETag for versioning
filerEntry := &filer.Entry{
FullPath: util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject)),
Attr: filer.Attr{
FileSize: dstEntry.Attributes.FileSize,
Mtime: time.Unix(dstEntry.Attributes.Mtime, 0),
Crtime: time.Unix(dstEntry.Attributes.Crtime, 0),
Mime: dstEntry.Attributes.Mime,
},
Chunks: dstEntry.Chunks,
}
etag = filer.ETagEntry(filerEntry)
if !strings.HasPrefix(etag, "\"") {
etag = "\"" + etag + "\""
}
dstEntry.Extended[s3_constants.ExtETagKey] = []byte(etag)
// Create version file
versionFileName := s3a.getVersionFileName(dstVersionId)
versionObjectPath := dstObject + ".versions/" + versionFileName
bucketDir := s3a.option.BucketsPath + "/" + dstBucket
if err := s3a.mkFile(bucketDir, versionObjectPath, dstEntry.Chunks, func(entry *filer_pb.Entry) {
entry.Attributes = dstEntry.Attributes
entry.Extended = dstEntry.Extended
}); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// Update the .versions directory metadata
err = s3a.updateLatestVersionInDirectory(dstBucket, dstObject, dstVersionId, versionFileName)
if err != nil {
glog.Errorf("CopyObjectHandler: failed to update latest version in directory: %v", err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// Set version ID in response header
w.Header().Set("x-amz-version-id", dstVersionId)
} else {
// For non-versioned destination, use regular copy
dstPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
dstDir, dstName := dstPath.DirAndName()
// Check if destination exists and remove it first (S3 copy overwrites)
if exists, _ := s3a.exists(dstDir, dstName, false); exists {
if err := s3a.rm(dstDir, dstName, false, false); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
}
// Create the new file
if err := s3a.mkFile(dstDir, dstName, dstEntry.Chunks, func(entry *filer_pb.Entry) {
entry.Attributes = dstEntry.Attributes
entry.Extended = dstEntry.Extended
}); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// Calculate ETag
filerEntry := &filer.Entry{
FullPath: dstPath,
Attr: filer.Attr{
FileSize: dstEntry.Attributes.FileSize,
Mtime: time.Unix(dstEntry.Attributes.Mtime, 0),
Crtime: time.Unix(dstEntry.Attributes.Crtime, 0),
Mime: dstEntry.Attributes.Mime,
},
Chunks: dstEntry.Chunks,
}
etag = filer.ETagEntry(filerEntry)
// Convert filer_pb.Entry to filer.Entry for ETag calculation
filerEntry := &filer.Entry{
FullPath: dstPath,
Attr: filer.Attr{
FileSize: dstEntry.Attributes.FileSize,
Mtime: time.Unix(dstEntry.Attributes.Mtime, 0),
Crtime: time.Unix(dstEntry.Attributes.Crtime, 0),
Mime: dstEntry.Attributes.Mime,
},
Chunks: dstEntry.Chunks,
}
setEtag(w, etag)
setEtag(w, filer.ETagEntry(filerEntry))
response := CopyObjectResult{
ETag: etag,
ETag: filer.ETagEntry(filerEntry),
LastModified: time.Now().UTC(),
}
@@ -288,18 +191,6 @@ func pathToBucketAndObject(path string) (bucket, object string) {
return parts[0], "/"
}
func pathToBucketObjectAndVersion(path string) (bucket, object, versionId string) {
// Parse versionId from query string if present
// Format: /bucket/object?versionId=version-id
if idx := strings.Index(path, "?versionId="); idx != -1 {
versionId = path[idx+len("?versionId="):] // dynamically calculate length
path = path[:idx]
}
bucket, object = pathToBucketAndObject(path)
return bucket, object, versionId
}
type CopyPartResult struct {
LastModified time.Time `xml:"LastModified"`
ETag string `xml:"ETag"`
@@ -317,7 +208,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
}
srcBucket, srcObject, srcVersionId := pathToBucketObjectAndVersion(cpSrcPath)
srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
// If source object is empty or bucket is empty, reply back invalid copy source.
if srcObject == "" || srcBucket == "" {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
@@ -348,40 +239,10 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
return
}
// Get detailed versioning state for source bucket
srcVersioningState, err := s3a.getVersioningState(srcBucket)
if err != nil {
glog.Errorf("Error checking versioning state for source bucket %s: %v", srcBucket, err)
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
return
}
// Get the source entry with version awareness based on versioning state
var entry *filer_pb.Entry
if srcVersionId != "" {
// Specific version requested - always use version-aware retrieval
entry, err = s3a.getSpecificObjectVersion(srcBucket, srcObject, srcVersionId)
} else if srcVersioningState == s3_constants.VersioningEnabled {
// Versioning enabled - get latest version from .versions directory
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
} else if srcVersioningState == s3_constants.VersioningSuspended {
// Versioning suspended - current object is stored as regular file ("null" version)
// Try regular file first, fall back to latest version if needed
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err = s3a.getEntry(dir, name)
if err != nil {
// If regular file doesn't exist, try latest version as fallback
glog.V(2).Infof("CopyObjectPart: regular file not found for suspended versioning, trying latest version")
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
}
} else {
// No versioning configured - use regular retrieval
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err = s3a.getEntry(dir, name)
}
// Get source entry
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
dir, name := srcPath.DirAndName()
entry, err := s3a.getEntry(dir, name)
if err != nil || entry.IsDirectory {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
return
+43 -158
View File
@@ -32,8 +32,8 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
// Check for specific version ID in query parameters
versionId := r.URL.Query().Get("versionId")
// Get detailed versioning state for proper handling of suspended vs enabled versioning
versioningState, err := s3a.getVersioningState(bucket)
// Check if versioning is enabled for the bucket
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
@@ -44,19 +44,14 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
return
}
versioningEnabled := (versioningState == s3_constants.VersioningEnabled)
versioningSuspended := (versioningState == s3_constants.VersioningSuspended)
versioningConfigured := (versioningState != "")
var auditLog *s3err.AccessLog
if s3err.Logger != nil {
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
}
if versioningConfigured {
// Handle versioned delete based on specific versioning state
if versioningEnabled {
// Handle versioned delete
if versionId != "" {
// Delete specific version (same for both enabled and suspended)
// Check object lock permissions before deleting specific version
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed); err != nil {
@@ -76,44 +71,19 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
// Set version ID in response header
w.Header().Set("x-amz-version-id", versionId)
} else {
// Delete without version ID - behavior depends on versioning state
if versioningEnabled {
// Enabled versioning: Create delete marker (logical delete)
// AWS S3 behavior: Delete marker creation is NOT blocked by object retention
// because it's a logical delete that doesn't actually remove the retained version
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
if err != nil {
glog.Errorf("Failed to create delete marker: %v", err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// Set delete marker version ID in response header
w.Header().Set("x-amz-version-id", deleteMarkerVersionId)
w.Header().Set("x-amz-delete-marker", "true")
} else if versioningSuspended {
// Suspended versioning: Actually delete the "null" version object
glog.V(2).Infof("DeleteObjectHandler: deleting null version for suspended versioning %s/%s", bucket, object)
// Check object lock permissions before deleting "null" version
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, "null", governanceBypassAllowed); err != nil {
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
return
}
// Delete the "null" version (the regular file)
err := s3a.deleteSpecificObjectVersion(bucket, object, "null")
if err != nil {
glog.Errorf("Failed to delete null version: %v", err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// Note: According to AWS S3 spec, suspended versioning should NOT return version ID headers
// The object is deleted but no version information is returned
// Create delete marker (logical delete)
// AWS S3 behavior: Delete marker creation is NOT blocked by object retention
// because it's a logical delete that doesn't actually remove the retained version
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
if err != nil {
glog.Errorf("Failed to create delete marker: %v", err)
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
// Set delete marker version ID in response header
w.Header().Set("x-amz-version-id", deleteMarkerVersionId)
w.Header().Set("x-amz-delete-marker", "true")
}
} else {
// Handle regular delete (non-versioned)
@@ -167,10 +137,8 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
// ObjectIdentifier represents an object to be deleted with its key name and optional version ID.
type ObjectIdentifier struct {
Key string `xml:"Key"`
VersionId string `xml:"VersionId,omitempty"`
DeleteMarker bool `xml:"DeleteMarker,omitempty"`
DeleteMarkerVersionId string `xml:"DeleteMarkerVersionId,omitempty"`
Key string `xml:"Key"`
VersionId string `xml:"VersionId,omitempty"`
}
// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
@@ -233,8 +201,8 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
}
// Get detailed versioning state for proper handling of suspended vs enabled versioning
versioningState, err := s3a.getVersioningState(bucket)
// Check if versioning is enabled for the bucket (needed for object lock checks)
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
@@ -245,10 +213,6 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
return
}
versioningEnabled := (versioningState == s3_constants.VersioningEnabled)
versioningSuspended := (versioningState == s3_constants.VersioningSuspended)
versioningConfigured := (versioningState != "")
s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
// delete file entries
@@ -258,7 +222,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
}
// Check object lock permissions before deletion (only for versioned buckets)
if versioningConfigured {
if versioningEnabled {
// Validate governance bypass for this specific object
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key)
if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, object.VersionId, governanceBypassAllowed); err != nil {
@@ -272,110 +236,31 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
continue
}
}
var deleteVersionId string
var isDeleteMarker bool
if versioningConfigured {
// Handle versioned delete based on specific versioning state
if object.VersionId != "" {
// Delete specific version (same for both enabled and suspended)
err := s3a.deleteSpecificObjectVersion(bucket, object.Key, object.VersionId)
if err != nil {
deleteErrors = append(deleteErrors, DeleteError{
Code: "",
Message: err.Error(),
Key: object.Key,
VersionId: object.VersionId,
})
continue
}
deleteVersionId = object.VersionId
} else {
// Delete without version ID - behavior depends on versioning state
if versioningEnabled {
// Enabled versioning: Create delete marker (logical delete)
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object.Key)
if err != nil {
deleteErrors = append(deleteErrors, DeleteError{
Code: "",
Message: err.Error(),
Key: object.Key,
VersionId: object.VersionId,
})
continue
}
deleteVersionId = deleteMarkerVersionId
isDeleteMarker = true
} else if versioningSuspended {
// Suspended versioning: Actually delete the "null" version object
glog.V(2).Infof("DeleteMultipleObjectsHandler: deleting null version for suspended versioning %s/%s", bucket, object.Key)
err := s3a.deleteSpecificObjectVersion(bucket, object.Key, "null")
if err != nil {
deleteErrors = append(deleteErrors, DeleteError{
Code: "",
Message: err.Error(),
Key: object.Key,
VersionId: "null",
})
continue
}
deleteVersionId = "null"
// Note: For suspended versioning, we don't set isDeleteMarker=true
// because we actually deleted the object, not created a delete marker
}
}
// Add to successful deletions with version info
deletedObject := ObjectIdentifier{
Key: object.Key,
VersionId: deleteVersionId,
DeleteMarker: isDeleteMarker,
}
// For delete markers, also set DeleteMarkerVersionId field
if isDeleteMarker {
deletedObject.DeleteMarkerVersionId = deleteVersionId
// Don't set VersionId for delete markers, use DeleteMarkerVersionId instead
deletedObject.VersionId = ""
}
if !deleteObjects.Quiet {
deletedObjects = append(deletedObjects, deletedObject)
}
if isDeleteMarker {
// For delete markers, we don't need to track directories for cleanup
continue
}
} else {
// Handle non-versioned delete (original logic)
lastSeparator := strings.LastIndex(object.Key, "/")
parentDirectoryPath, entryName, isDeleteData, isRecursive := "", object.Key, true, false
if lastSeparator > 0 && lastSeparator+1 < len(object.Key) {
entryName = object.Key[lastSeparator+1:]
parentDirectoryPath = "/" + object.Key[:lastSeparator]
}
parentDirectoryPath = fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, parentDirectoryPath)
err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
if err == nil {
directoriesWithDeletion[parentDirectoryPath]++
deletedObjects = append(deletedObjects, object)
} else if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
deletedObjects = append(deletedObjects, object)
} else {
delete(directoriesWithDeletion, parentDirectoryPath)
deleteErrors = append(deleteErrors, DeleteError{
Code: "",
Message: err.Error(),
Key: object.Key,
VersionId: object.VersionId,
})
}
lastSeparator := strings.LastIndex(object.Key, "/")
parentDirectoryPath, entryName, isDeleteData, isRecursive := "", object.Key, true, false
if lastSeparator > 0 && lastSeparator+1 < len(object.Key) {
entryName = object.Key[lastSeparator+1:]
parentDirectoryPath = "/" + object.Key[:lastSeparator]
}
parentDirectoryPath = fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, parentDirectoryPath)
err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
if err == nil {
directoriesWithDeletion[parentDirectoryPath]++
deletedObjects = append(deletedObjects, object)
} else if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
deletedObjects = append(deletedObjects, object)
} else {
delete(directoriesWithDeletion, parentDirectoryPath)
deleteErrors = append(deleteErrors, DeleteError{
Code: "",
Message: err.Error(),
Key: object.Key,
VersionId: object.VersionId,
})
}
if auditLog != nil {
auditLog.Key = object.Key
auditLog.Key = entryName
s3err.PostAccessLog(*auditLog)
}
}
+8 -141
View File
@@ -4,17 +4,16 @@ import (
"context"
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"io"
"net/http"
"net/url"
"strconv"
"strings"
)
type OptionalString struct {
@@ -148,32 +147,10 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
prefixEndsOnDelimiter: strings.HasSuffix(originalPrefix, "/") && len(originalMarker) == 0,
}
// Special case: when maxKeys = 0, return empty results immediately
if maxKeys == 0 {
response = ListBucketResult{
Name: bucket,
Prefix: originalPrefix,
Marker: originalMarker,
NextMarker: "",
MaxKeys: int(maxKeys),
Delimiter: delimiter,
IsTruncated: false,
Contents: contents,
CommonPrefixes: commonPrefixes,
}
if encodingTypeUrl {
response.EncodingType = s3.EncodingTypeUrl
}
return
}
// check filer
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
for {
empty := true
var lastEntryWasCommonPrefix bool
var lastCommonPrefixName string
nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
empty = false
dirName, entryName, prefixName := entryUrlEncode(dir, entry.Name, encodingTypeUrl)
@@ -181,7 +158,6 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
if entry.IsDirectoryKeyObject() {
contents = append(contents, newListEntry(entry, "", dirName, entryName, bucketPrefix, fetchOwner, true, false))
cursor.maxKeys--
lastEntryWasCommonPrefix = false
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
} else if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
commonPrefixes = append(commonPrefixes, PrefixEntry{
@@ -189,8 +165,6 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
})
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
cursor.maxKeys--
lastEntryWasCommonPrefix = true
lastCommonPrefixName = entry.Name
}
} else {
var delimiterFound bool
@@ -221,15 +195,12 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
})
cursor.maxKeys--
delimiterFound = true
lastEntryWasCommonPrefix = true
lastCommonPrefixName = delimitedPath[0]
}
}
}
if !delimiterFound {
contents = append(contents, newListEntry(entry, "", dirName, entryName, bucketPrefix, fetchOwner, false, false))
cursor.maxKeys--
lastEntryWasCommonPrefix = false
}
}
})
@@ -237,21 +208,10 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
return doErr
}
// Adjust nextMarker for CommonPrefixes to include trailing slash (AWS S3 compliance)
if cursor.isTruncated && lastEntryWasCommonPrefix && lastCommonPrefixName != "" {
// For CommonPrefixes, NextMarker should include the trailing slash
if requestDir != "" {
nextMarker = requestDir + "/" + lastCommonPrefixName + "/"
} else {
nextMarker = lastCommonPrefixName + "/"
}
} else if cursor.isTruncated {
if cursor.isTruncated {
if requestDir != "" {
nextMarker = requestDir + "/" + nextMarker
}
}
if cursor.isTruncated {
break
} else if empty || strings.HasSuffix(originalPrefix, "/") {
nextMarker = ""
@@ -396,9 +356,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
return
}
// Track .versions directories found in this directory for later processing
var versionsDirs []string
for {
resp, recvErr := stream.Recv()
if recvErr != nil {
@@ -410,18 +367,8 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
}
}
if cursor.maxKeys <= 0 {
// Check if there are more entries available by trying to peek at the next one
_, nextRecvErr := stream.Recv()
if nextRecvErr == nil {
// There is another entry available, so we're truncated
cursor.isTruncated = true
} else if nextRecvErr != io.EOF {
// Some other error occurred
err = fmt.Errorf("peeking next entry: %v", nextRecvErr)
return
}
// If nextRecvErr == io.EOF, there are no more entries, so isTruncated remains false
break
cursor.isTruncated = true
continue
}
entry := resp.Entry
nextMarker = entry.Name
@@ -439,14 +386,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
continue
}
// Skip .versions directories in regular list operations but track them for logical object creation
if strings.HasSuffix(entry.Name, ".versions") {
glog.V(4).Infof("Found .versions directory: %s", entry.Name)
versionsDirs = append(versionsDirs, entry.Name)
continue
}
if delimiter != "/" || cursor.prefixEndsOnDelimiter {
if cursor.prefixEndsOnDelimiter {
cursor.prefixEndsOnDelimiter = false
@@ -486,49 +425,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
cursor.prefixEndsOnDelimiter = false
}
}
// After processing all regular entries, handle versioned objects
// Create logical entries for objects that have .versions directories
for _, versionsDir := range versionsDirs {
if cursor.maxKeys <= 0 {
// We have versioned objects remaining but can't process them due to limit
cursor.isTruncated = true
break
}
// Extract object name from .versions directory name (remove .versions suffix)
baseObjectName := strings.TrimSuffix(versionsDir, ".versions")
// Construct full object path relative to bucket
// dir is something like "/buckets/sea-test-1/Veeam/Backup/vbr/Config"
// we need to get the path relative to bucket: "Veeam/Backup/vbr/Config/Owner"
bucketPath := strings.TrimPrefix(dir, s3a.option.BucketsPath+"/")
bucketName := strings.Split(bucketPath, "/")[0]
// Remove bucket name from path to get directory within bucket
bucketRelativePath := strings.Join(strings.Split(bucketPath, "/")[1:], "/")
var fullObjectPath string
if bucketRelativePath == "" {
// Object is at bucket root
fullObjectPath = baseObjectName
} else {
// Object is in subdirectory
fullObjectPath = bucketRelativePath + "/" + baseObjectName
}
glog.V(4).Infof("Processing versioned object: baseObjectName=%s, bucketRelativePath=%s, fullObjectPath=%s",
baseObjectName, bucketRelativePath, fullObjectPath)
// Get the latest version information for this object
if latestVersionEntry, latestVersionErr := s3a.getLatestVersionEntryForListOperation(bucketName, fullObjectPath); latestVersionErr == nil {
glog.V(4).Infof("Creating logical entry for versioned object: %s", fullObjectPath)
eachEntryFn(dir, latestVersionEntry)
} else {
glog.V(4).Infof("Failed to get latest version for %s: %v", fullObjectPath, latestVersionErr)
}
}
return
}
@@ -617,32 +513,3 @@ func (s3a *S3ApiServer) ensureDirectoryAllEmpty(filerClient filer_pb.SeaweedFile
return true, nil
}
// getLatestVersionEntryForListOperation gets the latest version of an object and creates a logical entry for list operations
// This is used to show versioned objects as logical object names in regular list operations
func (s3a *S3ApiServer) getLatestVersionEntryForListOperation(bucket, object string) (*filer_pb.Entry, error) {
// Get the latest version entry
latestVersionEntry, err := s3a.getLatestObjectVersion(bucket, object)
if err != nil {
return nil, fmt.Errorf("failed to get latest version: %w", err)
}
// Check if this is a delete marker (should not be shown in regular list)
if latestVersionEntry.Extended != nil {
if deleteMarker, exists := latestVersionEntry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
return nil, fmt.Errorf("latest version is a delete marker")
}
}
// Create a logical entry that appears to be stored at the object path (not the versioned path)
// This allows the list operation to show the logical object name while preserving all metadata
logicalEntry := &filer_pb.Entry{
Name: strings.TrimPrefix(object, "/"),
IsDirectory: false,
Attributes: latestVersionEntry.Attributes,
Extended: latestVersionEntry.Extended,
Chunks: latestVersionEntry.Chunks,
}
return logicalEntry, nil
}
@@ -3,7 +3,6 @@ package s3api
import (
"crypto/sha1"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
@@ -42,7 +41,7 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
// Check if versioning is enabled for the bucket (needed for object lock)
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
@@ -112,7 +111,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
return
}
response, errCode := s3a.completeMultipartUpload(r, &s3.CompleteMultipartUploadInput{
response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket),
Key: objectKey(aws.String(object)),
UploadId: aws.String(uploadID),
+9 -182
View File
@@ -90,18 +90,15 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
entry.Content, _ = io.ReadAll(r.Body)
}
entry.Attributes.Mime = objectContentType
// Set object owner for directory objects (same as regular objects)
s3a.setObjectOwnerFromRequest(r, entry)
}); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
} else {
// Get detailed versioning state for the bucket
versioningState, err := s3a.getVersioningState(bucket)
// Check if versioning is enabled for the bucket
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
if err != nil {
if errors.Is(err, filer_pb.ErrNotFound) {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
@@ -110,10 +107,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
return
}
versioningEnabled := (versioningState == s3_constants.VersioningEnabled)
versioningConfigured := (versioningState != "")
glog.V(1).Infof("PutObjectHandler: bucket %s, object %s, versioningState=%s", bucket, object, versioningState)
glog.V(1).Infof("PutObjectHandler: bucket %s, object %s, versioningEnabled=%v", bucket, object, versioningEnabled)
// Validate object lock headers before processing
if err := s3a.validateObjectLockHeaders(r, versioningEnabled); err != nil {
@@ -124,7 +118,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
// For non-versioned buckets, check if existing object has object lock protections
// that would prevent overwrite (PUT operations overwrite existing objects in non-versioned buckets)
if !versioningConfigured {
if !versioningEnabled {
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
if err := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); err != nil {
glog.V(2).Infof("PutObjectHandler: object lock permissions check failed for %s/%s: %v", bucket, object, err)
@@ -133,8 +127,8 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
}
}
if versioningState == s3_constants.VersioningEnabled {
// Handle enabled versioning - create new versions with real version IDs
if versioningEnabled {
// Handle versioned PUT
glog.V(1).Infof("PutObjectHandler: using versioned PUT for %s/%s", bucket, object)
versionId, etag, errCode := s3a.putVersionedObject(r, bucket, object, dataReader, objectContentType)
if errCode != s3err.ErrNone {
@@ -147,24 +141,10 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
w.Header().Set("x-amz-version-id", versionId)
}
// Set ETag in response
setEtag(w, etag)
} else if versioningState == s3_constants.VersioningSuspended {
// Handle suspended versioning - overwrite with "null" version ID but preserve existing versions
glog.V(1).Infof("PutObjectHandler: using suspended versioning PUT for %s/%s", bucket, object)
etag, errCode := s3a.putSuspendedVersioningObject(r, bucket, object, dataReader, objectContentType)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
// Note: Suspended versioning should NOT return x-amz-version-id header according to AWS S3 spec
// The object is stored with "null" version internally but no version header is returned
// Set ETag in response
setEtag(w, etag)
} else {
// Handle regular PUT (never configured versioning)
// Handle regular PUT (non-versioned)
glog.V(1).Infof("PutObjectHandler: using regular PUT for %s/%s", bucket, object)
uploadUrl := s3a.toFilerUrl(bucket, object)
if objectContentType == "" {
@@ -178,7 +158,6 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
return
}
// No version ID header for never-configured versioning
setEtag(w, etag)
}
}
@@ -216,14 +195,6 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader
proxyReq.Header.Add(header, value)
}
}
// Set object owner header for filer to extract
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
if amzAccountId != "" {
proxyReq.Header.Set(s3_constants.ExtAmzOwnerKey, amzAccountId)
glog.V(2).Infof("putToFiler: setting owner header %s for object %s", amzAccountId, uploadUrl)
}
// ensure that the Authorization header is overriding any previous
// Authorization header which might be already present in proxyReq
s3a.maybeAddFilerJwtAuthorization(proxyReq, true)
@@ -255,8 +226,8 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader
glog.Errorf("upload to filer error: %v", ret.Error)
return "", filerErrorToS3Error(ret.Error)
}
stats_collect.RecordBucketActiveTime(bucket)
stats_collect.S3BucketTrafficReceivedBytesCounter.WithLabelValues(bucket).Add(float64(ret.Size))
return etag, s3err.ErrNone
}
@@ -301,149 +272,8 @@ func (s3a *S3ApiServer) maybeGetFilerJwtAuthorizationToken(isWrite bool) string
return string(encodedJwt)
}
// setObjectOwnerFromRequest sets the object owner metadata based on the authenticated user
func (s3a *S3ApiServer) setObjectOwnerFromRequest(r *http.Request, entry *filer_pb.Entry) {
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
if amzAccountId != "" {
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(amzAccountId)
glog.V(2).Infof("setObjectOwnerFromRequest: set object owner to %s", amzAccountId)
}
}
// putVersionedObject handles PUT operations for versioned buckets using the new layout
// where all versions (including latest) are stored in the .versions directory
func (s3a *S3ApiServer) putSuspendedVersioningObject(r *http.Request, bucket, object string, dataReader io.Reader, objectContentType string) (etag string, errCode s3err.ErrorCode) {
// For suspended versioning, store as regular object (version ID "null") but preserve existing versions
glog.V(2).Infof("putSuspendedVersioningObject: creating null version for %s/%s", bucket, object)
uploadUrl := s3a.toFilerUrl(bucket, object)
if objectContentType == "" {
dataReader = mimeDetect(r, dataReader)
}
etag, errCode = s3a.putToFiler(r, uploadUrl, dataReader, "", bucket)
if errCode != s3err.ErrNone {
glog.Errorf("putSuspendedVersioningObject: failed to upload object: %v", errCode)
return "", errCode
}
// Get the uploaded entry to add version metadata indicating this is "null" version
bucketDir := s3a.option.BucketsPath + "/" + bucket
entry, err := s3a.getEntry(bucketDir, object)
if err != nil {
glog.Errorf("putSuspendedVersioningObject: failed to get object entry: %v", err)
return "", s3err.ErrInternalError
}
// Add metadata to indicate this is a "null" version for suspended versioning
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.ExtVersionIdKey] = []byte("null")
// Set object owner for suspended versioning objects
s3a.setObjectOwnerFromRequest(r, entry)
// Extract and store object lock metadata from request headers (if any)
if err := s3a.extractObjectLockMetadataFromRequest(r, entry); err != nil {
glog.Errorf("putSuspendedVersioningObject: failed to extract object lock metadata: %v", err)
return "", s3err.ErrInvalidRequest
}
// Update the entry with metadata
err = s3a.mkFile(bucketDir, object, entry.Chunks, func(updatedEntry *filer_pb.Entry) {
updatedEntry.Extended = entry.Extended
updatedEntry.Attributes = entry.Attributes
updatedEntry.Chunks = entry.Chunks
})
if err != nil {
glog.Errorf("putSuspendedVersioningObject: failed to update object metadata: %v", err)
return "", s3err.ErrInternalError
}
// Update all existing versions/delete markers to set IsLatest=false since "null" is now latest
err = s3a.updateIsLatestFlagsForSuspendedVersioning(bucket, object)
if err != nil {
glog.Warningf("putSuspendedVersioningObject: failed to update IsLatest flags: %v", err)
// Don't fail the request, but log the warning
}
glog.V(2).Infof("putSuspendedVersioningObject: successfully created null version for %s/%s", bucket, object)
return etag, s3err.ErrNone
}
// updateIsLatestFlagsForSuspendedVersioning sets IsLatest=false on all existing versions/delete markers
// when a new "null" version becomes the latest during suspended versioning
func (s3a *S3ApiServer) updateIsLatestFlagsForSuspendedVersioning(bucket, object string) error {
bucketDir := s3a.option.BucketsPath + "/" + bucket
versionsObjectPath := object + ".versions"
versionsDir := bucketDir + "/" + versionsObjectPath
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: updating flags for %s%s", bucket, object)
// Check if .versions directory exists
_, err := s3a.getEntry(bucketDir, versionsObjectPath)
if err != nil {
// No .versions directory exists, nothing to update
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: no .versions directory for %s%s", bucket, object)
return nil
}
// List all entries in .versions directory
entries, _, err := s3a.list(versionsDir, "", "", false, 1000)
if err != nil {
return fmt.Errorf("failed to list versions directory: %v", err)
}
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: found %d entries to update", len(entries))
// Update each version/delete marker to set IsLatest=false
for _, entry := range entries {
if entry.Extended == nil {
continue
}
// Check if this entry has a version ID (it should be a version or delete marker)
versionIdBytes, hasVersionId := entry.Extended[s3_constants.ExtVersionIdKey]
if !hasVersionId {
continue
}
versionId := string(versionIdBytes)
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: setting IsLatest=false for version %s", versionId)
// Update the entry to set IsLatest=false (we don't explicitly store this flag,
// it's determined by comparison with latest version metadata)
// We need to clear the latest version metadata from the .versions directory
// so that our getObjectVersionList function will correctly show IsLatest=false
}
// Clear the latest version metadata from .versions directory since "null" is now latest
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
if err == nil && versionsEntry.Extended != nil {
// Remove latest version metadata so all versions show IsLatest=false
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionIdKey)
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionFileNameKey)
// Update the .versions directory entry
err = s3a.mkFile(bucketDir, versionsObjectPath, versionsEntry.Chunks, func(updatedEntry *filer_pb.Entry) {
updatedEntry.Extended = versionsEntry.Extended
updatedEntry.Attributes = versionsEntry.Attributes
updatedEntry.Chunks = versionsEntry.Chunks
})
if err != nil {
return fmt.Errorf("failed to update .versions directory metadata: %v", err)
}
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: cleared latest version metadata for %s%s", bucket, object)
}
return nil
}
func (s3a *S3ApiServer) putVersionedObject(r *http.Request, bucket, object string, dataReader io.Reader, objectContentType string) (versionId string, etag string, errCode s3err.ErrorCode) {
// Generate version ID
versionId = generateVersionId()
@@ -492,9 +322,6 @@ func (s3a *S3ApiServer) putVersionedObject(r *http.Request, bucket, object strin
}
versionEntry.Extended[s3_constants.ExtETagKey] = []byte(etag)
// Set object owner for versioned objects
s3a.setObjectOwnerFromRequest(r, versionEntry)
// Extract and store object lock metadata from request headers
if err := s3a.extractObjectLockMetadataFromRequest(r, versionEntry); err != nil {
glog.Errorf("putVersionedObject: failed to extract object lock metadata: %v", err)
+21
View File
@@ -0,0 +1,21 @@
package s3api
import (
"net/http"
)
// GetObjectAclHandler Get object ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html
func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
// PutObjectAclHandler Put object ACL
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html
func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
+89 -488
View File
@@ -2,6 +2,7 @@ package s3api
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"fmt"
@@ -19,31 +20,18 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
// S3ListObjectVersionsResult - Custom struct for S3 list-object-versions response
// This avoids conflicts with the XSD generated ListVersionsResult struct
// and ensures proper separation of versions and delete markers into arrays
type S3ListObjectVersionsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult"`
Name string `xml:"Name"`
Prefix string `xml:"Prefix,omitempty"`
KeyMarker string `xml:"KeyMarker,omitempty"`
VersionIdMarker string `xml:"VersionIdMarker,omitempty"`
NextKeyMarker string `xml:"NextKeyMarker,omitempty"`
NextVersionIdMarker string `xml:"NextVersionIdMarker,omitempty"`
MaxKeys int `xml:"MaxKeys"`
Delimiter string `xml:"Delimiter,omitempty"`
IsTruncated bool `xml:"IsTruncated"`
// These are the critical fields - arrays instead of single elements
Versions []VersionEntry `xml:"Version,omitempty"` // Array for versions
DeleteMarkers []DeleteMarkerEntry `xml:"DeleteMarker,omitempty"` // Array for delete markers
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
EncodingType string `xml:"EncodingType,omitempty"`
// ObjectVersion represents a version of an S3 object
type ObjectVersion struct {
VersionId string
IsLatest bool
IsDeleteMarker bool
LastModified time.Time
ETag string
Size int64
Entry *filer_pb.Entry
}
// Original struct - keeping for compatibility but will use S3ListObjectVersionsResult for XML response
// ListObjectVersionsResult represents the response for ListObjectVersions
type ListObjectVersionsResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult"`
Name string `xml:"Name"`
@@ -60,37 +48,20 @@ type ListObjectVersionsResult struct {
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
}
// ObjectVersion represents a version of an S3 object
type ObjectVersion struct {
VersionId string
IsLatest bool
IsDeleteMarker bool
LastModified time.Time
ETag string
Size int64
Entry *filer_pb.Entry
}
// generateVersionId creates a unique version ID that preserves chronological order
// generateVersionId creates a unique version ID
func generateVersionId() string {
// Use nanosecond timestamp to ensure chronological ordering
// Format as 16-digit hex (first 16 chars of version ID)
now := time.Now().UnixNano()
timestampHex := fmt.Sprintf("%016x", now)
// Generate random 8 bytes for uniqueness (last 16 chars of version ID)
randBytes := make([]byte, 8)
// Generate a random 16-byte value
randBytes := make([]byte, 16)
if _, err := rand.Read(randBytes); err != nil {
glog.Errorf("Failed to generate random bytes for version ID: %v", err)
// Fallback to timestamp-only if random generation fails
return timestampHex + "0000000000000000"
return ""
}
// Combine timestamp (16 chars) + random (16 chars) = 32 chars total
randomHex := hex.EncodeToString(randBytes)
versionId := timestampHex + randomHex
// Hash with current timestamp for uniqueness
hash := sha256.Sum256(append(randBytes, []byte(fmt.Sprintf("%d", time.Now().UnixNano()))...))
return versionId
// Return first 32 characters of hex string (same length as AWS S3 version IDs)
return hex.EncodeToString(hash[:])[:32]
}
// getVersionedObjectDir returns the directory path for storing object versions
@@ -148,23 +119,62 @@ func (s3a *S3ApiServer) createDeleteMarker(bucket, object string) (string, error
}
// listObjectVersions lists all versions of an object
func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int) (*S3ListObjectVersionsResult, error) {
func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int) (*ListObjectVersionsResult, error) {
var allVersions []interface{} // Can contain VersionEntry or DeleteMarkerEntry
// Track objects that have been processed to avoid duplicates
processedObjects := make(map[string]bool)
// Track version IDs globally to prevent duplicates throughout the listing
seenVersionIds := make(map[string]bool)
// Recursively find all .versions directories in the bucket
bucketPath := path.Join(s3a.option.BucketsPath, bucket)
err := s3a.findVersionsRecursively(bucketPath, "", &allVersions, processedObjects, seenVersionIds, bucket, prefix)
// List all entries in bucket
entries, _, err := s3a.list(path.Join(s3a.option.BucketsPath, bucket), prefix, keyMarker, false, uint32(maxKeys*2))
if err != nil {
return nil, err
}
// Sort by key, then by LastModified (newest first), then by VersionId for deterministic ordering
// For each entry, check if it's a .versions directory
for _, entry := range entries {
if !entry.IsDirectory {
continue
}
// Check if this is a .versions directory
if !strings.HasSuffix(entry.Name, ".versions") {
continue
}
// Extract object name from .versions directory name
objectKey := strings.TrimSuffix(entry.Name, ".versions")
versions, err := s3a.getObjectVersionList(bucket, objectKey)
if err != nil {
glog.Warningf("Failed to get versions for object %s: %v", objectKey, err)
continue
}
for _, version := range versions {
if version.IsDeleteMarker {
deleteMarker := &DeleteMarkerEntry{
Key: objectKey,
VersionId: version.VersionId,
IsLatest: version.IsLatest,
LastModified: version.LastModified,
Owner: CanonicalUser{ID: "unknown", DisplayName: "unknown"},
}
allVersions = append(allVersions, deleteMarker)
} else {
versionEntry := &VersionEntry{
Key: objectKey,
VersionId: version.VersionId,
IsLatest: version.IsLatest,
LastModified: version.LastModified,
ETag: version.ETag,
Size: version.Size,
Owner: CanonicalUser{ID: "unknown", DisplayName: "unknown"},
StorageClass: "STANDARD",
}
allVersions = append(allVersions, versionEntry)
}
}
}
// Sort by key, then by LastModified and VersionId
sort.Slice(allVersions, func(i, j int) bool {
var keyI, keyJ string
var lastModifiedI, lastModifiedJ time.Time
@@ -192,24 +202,17 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
versionIdJ = v.VersionId
}
// First sort by object key
if keyI != keyJ {
return keyI < keyJ
}
// Then by modification time (newest first) - but use nanosecond precision for ties
timeDiff := lastModifiedI.Sub(lastModifiedJ)
if timeDiff.Abs() > time.Millisecond {
if !lastModifiedI.Equal(lastModifiedJ) {
return lastModifiedI.After(lastModifiedJ)
}
// For very close timestamps (within 1ms), use version ID for deterministic ordering
// Sort version IDs in reverse lexicographic order to maintain newest-first semantics
return versionIdI > versionIdJ
return versionIdI < versionIdJ
})
// Build result using S3ListObjectVersionsResult to avoid conflicts with XSD structs
result := &S3ListObjectVersionsResult{
// Build result
result := &ListObjectVersionsResult{
Name: bucket,
Prefix: prefix,
KeyMarker: keyMarker,
@@ -234,10 +237,6 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
}
}
// Always initialize empty slices so boto3 gets the expected fields even when empty
result.Versions = make([]VersionEntry, 0)
result.DeleteMarkers = make([]DeleteMarkerEntry, 0)
// Add versions to result
for _, version := range allVersions {
switch v := version.(type) {
@@ -251,172 +250,6 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
return result, nil
}
// findVersionsRecursively searches for all .versions directories and regular files recursively
func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string, allVersions *[]interface{}, processedObjects map[string]bool, seenVersionIds map[string]bool, bucket, prefix string) error {
// List entries in current directory
entries, _, err := s3a.list(currentPath, "", "", false, 1000)
if err != nil {
return err
}
for _, entry := range entries {
entryPath := path.Join(relativePath, entry.Name)
// Skip if this doesn't match the prefix filter
if prefix != "" {
normalizedPrefix := strings.TrimPrefix(prefix, "/")
if normalizedPrefix != "" {
// An entry is a candidate if:
// 1. Its path is prefixed by normalizedPrefix.
// 2. It is a directory that is a prefix of normalizedPrefix (so we can descend into it).
isPrefixed := strings.HasPrefix(entryPath, normalizedPrefix)
if !isPrefixed && entry.IsDirectory {
// For directories, also check with a trailing slash.
isPrefixed = strings.HasPrefix(entryPath+"/", normalizedPrefix)
}
canDescend := entry.IsDirectory && strings.HasPrefix(normalizedPrefix, entryPath)
if !isPrefixed && !canDescend {
continue
}
}
}
if entry.IsDirectory {
// Skip .uploads directory (multipart upload temporary files)
if strings.HasPrefix(entry.Name, ".uploads") {
continue
}
// Check if this is a .versions directory
if strings.HasSuffix(entry.Name, ".versions") {
// Extract object name from .versions directory name
objectKey := strings.TrimSuffix(entryPath, ".versions")
processedObjects[objectKey] = true
glog.V(2).Infof("findVersionsRecursively: found .versions directory for object %s", objectKey)
versions, err := s3a.getObjectVersionList(bucket, objectKey)
if err != nil {
glog.Warningf("Failed to get versions for object %s: %v", objectKey, err)
continue
}
for _, version := range versions {
// Check for duplicate version IDs and skip if already seen
versionKey := objectKey + ":" + version.VersionId
if seenVersionIds[versionKey] {
glog.Warningf("findVersionsRecursively: duplicate version %s for object %s detected, skipping", version.VersionId, objectKey)
continue
}
seenVersionIds[versionKey] = true
if version.IsDeleteMarker {
deleteMarker := &DeleteMarkerEntry{
Key: objectKey,
VersionId: version.VersionId,
IsLatest: version.IsLatest,
LastModified: version.LastModified,
Owner: s3a.getObjectOwnerFromVersion(version, bucket, objectKey),
}
*allVersions = append(*allVersions, deleteMarker)
} else {
versionEntry := &VersionEntry{
Key: objectKey,
VersionId: version.VersionId,
IsLatest: version.IsLatest,
LastModified: version.LastModified,
ETag: version.ETag,
Size: version.Size,
Owner: s3a.getObjectOwnerFromVersion(version, bucket, objectKey),
StorageClass: "STANDARD",
}
*allVersions = append(*allVersions, versionEntry)
}
}
} else {
// This is a regular directory - check if it's an explicit S3 directory object
// Only include directories that were explicitly created via S3 API (have FolderMimeType)
// This excludes implicit directories created when uploading files like "test1/a"
if entry.Attributes.Mime == s3_constants.FolderMimeType {
directoryKey := entryPath
if !strings.HasSuffix(directoryKey, "/") {
directoryKey += "/"
}
// Add directory as a version entry with VersionId "null" (following S3/Minio behavior)
glog.V(2).Infof("findVersionsRecursively: found explicit S3 directory %s", directoryKey)
// Calculate ETag for empty directory
directoryETag := "\"d41d8cd98f00b204e9800998ecf8427e\""
versionEntry := &VersionEntry{
Key: directoryKey,
VersionId: "null",
IsLatest: true,
LastModified: time.Unix(entry.Attributes.Mtime, 0),
ETag: directoryETag,
Size: 0, // Directories have size 0
Owner: s3a.getObjectOwnerFromEntry(entry),
StorageClass: "STANDARD",
}
*allVersions = append(*allVersions, versionEntry)
}
// Recursively search subdirectories (regardless of whether they're explicit or implicit)
fullPath := path.Join(currentPath, entry.Name)
err := s3a.findVersionsRecursively(fullPath, entryPath, allVersions, processedObjects, seenVersionIds, bucket, prefix)
if err != nil {
glog.Warningf("Error searching subdirectory %s: %v", entryPath, err)
continue
}
}
} else {
// This is a regular file - check if it's a pre-versioning object
objectKey := entryPath
// Skip if this object already has a .versions directory (already processed)
if processedObjects[objectKey] {
continue
}
// This is a pre-versioning object - treat it as a version with VersionId="null"
glog.V(2).Infof("findVersionsRecursively: found pre-versioning object %s", objectKey)
// Check if this null version should be marked as latest
// It's only latest if there's no .versions directory OR no latest version metadata
isLatest := true
versionsObjectPath := objectKey + ".versions"
if versionsEntry, err := s3a.getEntry(currentPath, versionsObjectPath); err == nil {
// .versions directory exists, check if there's latest version metadata
if versionsEntry.Extended != nil {
if _, hasLatest := versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey]; hasLatest {
// There is a latest version in the .versions directory, so null is not latest
isLatest = false
glog.V(2).Infof("findVersionsRecursively: null version for %s is not latest due to versioned objects", objectKey)
}
}
}
etag := s3a.calculateETagFromChunks(entry.Chunks)
versionEntry := &VersionEntry{
Key: objectKey,
VersionId: "null",
IsLatest: isLatest,
LastModified: time.Unix(entry.Attributes.Mtime, 0),
ETag: etag,
Size: int64(entry.Attributes.FileSize),
Owner: s3a.getObjectOwnerFromEntry(entry),
StorageClass: "STANDARD",
}
*allVersions = append(*allVersions, versionEntry)
}
}
return nil
}
// getObjectVersionList returns all versions of a specific object
func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVersion, error) {
var versions []*ObjectVersion
@@ -454,9 +287,6 @@ func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVe
glog.V(2).Infof("getObjectVersionList: found %d entries in versions directory", len(entries))
// Use a map to detect and prevent duplicate version IDs
seenVersionIds := make(map[string]bool)
for i, entry := range entries {
if entry.Extended == nil {
glog.V(2).Infof("getObjectVersionList: entry %d has no Extended metadata, skipping", i)
@@ -471,13 +301,6 @@ func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVe
versionId := string(versionIdBytes)
// Check for duplicate version IDs and skip if already seen
if seenVersionIds[versionId] {
glog.Warningf("getObjectVersionList: duplicate version ID %s detected for object %s/%s, skipping", versionId, bucket, object)
continue
}
seenVersionIds[versionId] = true
// Check if this version is the latest by comparing with directory metadata
isLatest := (versionId == latestVersionId)
@@ -508,9 +331,12 @@ func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVe
versions = append(versions, version)
}
// Don't sort here - let the main listObjectVersions function handle sorting consistently
// Sort by modification time (newest first)
sort.Slice(versions, func(i, j int) bool {
return versions[i].LastModified.After(versions[j].LastModified)
})
glog.V(2).Infof("getObjectVersionList: returning %d total versions for %s/%s (after deduplication from %d entries)", len(versions), bucket, object, len(entries))
glog.V(2).Infof("getObjectVersionList: returning %d total versions for %s/%s", len(versions), bucket, object)
for i, version := range versions {
glog.V(2).Infof("getObjectVersionList: version %d: %s (isLatest=%v, isDeleteMarker=%v)", i, version.VersionId, version.IsLatest, version.IsDeleteMarker)
}
@@ -540,16 +366,6 @@ func (s3a *S3ApiServer) getSpecificObjectVersion(bucket, object, versionId strin
return s3a.getEntry(path.Join(s3a.option.BucketsPath, bucket), strings.TrimPrefix(object, "/"))
}
if versionId == "null" {
// "null" version ID refers to pre-versioning objects stored as regular files
bucketDir := s3a.option.BucketsPath + "/" + bucket
entry, err := s3a.getEntry(bucketDir, object)
if err != nil {
return nil, fmt.Errorf("null version object %s not found: %v", object, err)
}
return entry, nil
}
// Get specific version from .versions directory
versionsDir := s3a.getVersionedObjectDir(bucket, object)
versionFile := s3a.getVersionFileName(versionId)
@@ -568,153 +384,25 @@ func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId st
return fmt.Errorf("version ID is required for version-specific deletion")
}
if versionId == "null" {
// Delete "null" version (pre-versioning object stored as regular file)
bucketDir := s3a.option.BucketsPath + "/" + bucket
cleanObject := strings.TrimPrefix(object, "/")
// Check if the object exists
_, err := s3a.getEntry(bucketDir, cleanObject)
if err != nil {
// Object doesn't exist - this is OK for delete operations (idempotent)
glog.V(2).Infof("deleteSpecificObjectVersion: null version object %s already deleted or doesn't exist", cleanObject)
return nil
}
// Delete the regular file
deleteErr := s3a.rm(bucketDir, cleanObject, true, false)
if deleteErr != nil {
// Check if file was already deleted by another process
if _, checkErr := s3a.getEntry(bucketDir, cleanObject); checkErr != nil {
// File doesn't exist anymore, deletion was successful
return nil
}
return fmt.Errorf("failed to delete null version %s: %v", cleanObject, deleteErr)
}
return nil
}
versionsDir := s3a.getVersionedObjectDir(bucket, object)
versionFile := s3a.getVersionFileName(versionId)
// Check if this is the latest version before attempting deletion (for potential metadata update)
versionsEntry, dirErr := s3a.getEntry(path.Join(s3a.option.BucketsPath, bucket), object+".versions")
isLatestVersion := false
if dirErr == nil && versionsEntry.Extended != nil {
if latestVersionIdBytes, hasLatest := versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey]; hasLatest {
isLatestVersion = (string(latestVersionIdBytes) == versionId)
}
// Delete the specific version from .versions directory
_, err := s3a.getEntry(versionsDir, versionFile)
if err != nil {
return fmt.Errorf("version %s not found: %v", versionId, err)
}
// Attempt to delete the version file
// Note: We don't check if the file exists first to avoid race conditions
// The deletion operation should be idempotent
// Version exists, delete it
deleteErr := s3a.rm(versionsDir, versionFile, true, false)
if deleteErr != nil {
// Check if file was already deleted by another process (race condition handling)
// Check if file was already deleted by another process
if _, checkErr := s3a.getEntry(versionsDir, versionFile); checkErr != nil {
// File doesn't exist anymore, deletion was successful (another thread deleted it)
glog.V(2).Infof("deleteSpecificObjectVersion: version %s for %s%s already deleted by another process", versionId, bucket, object)
// File doesn't exist anymore, deletion was successful
return nil
}
// File still exists but deletion failed for another reason
return fmt.Errorf("failed to delete version %s: %v", versionId, deleteErr)
}
// If we deleted the latest version, update the .versions directory metadata to point to the new latest
if isLatestVersion {
err := s3a.updateLatestVersionAfterDeletion(bucket, object)
if err != nil {
glog.Warningf("deleteSpecificObjectVersion: failed to update latest version after deletion: %v", err)
// Don't return error since the deletion was successful
}
}
return nil
}
// updateLatestVersionAfterDeletion finds the new latest version after deleting the current latest
func (s3a *S3ApiServer) updateLatestVersionAfterDeletion(bucket, object string) error {
bucketDir := s3a.option.BucketsPath + "/" + bucket
cleanObject := strings.TrimPrefix(object, "/")
versionsObjectPath := cleanObject + ".versions"
versionsDir := bucketDir + "/" + versionsObjectPath
glog.V(1).Infof("updateLatestVersionAfterDeletion: updating latest version for %s/%s, listing %s", bucket, object, versionsDir)
// List all remaining version files in the .versions directory
entries, _, err := s3a.list(versionsDir, "", "", false, 1000)
if err != nil {
glog.Errorf("updateLatestVersionAfterDeletion: failed to list versions in %s: %v", versionsDir, err)
return fmt.Errorf("failed to list versions: %v", err)
}
glog.V(1).Infof("updateLatestVersionAfterDeletion: found %d entries in %s", len(entries), versionsDir)
// Find the most recent remaining version (latest timestamp in version ID)
var latestVersionId string
var latestVersionFileName string
for _, entry := range entries {
if entry.Extended == nil {
continue
}
versionIdBytes, hasVersionId := entry.Extended[s3_constants.ExtVersionIdKey]
if !hasVersionId {
continue
}
versionId := string(versionIdBytes)
// Skip delete markers when finding latest content version
isDeleteMarkerBytes, _ := entry.Extended[s3_constants.ExtDeleteMarkerKey]
if string(isDeleteMarkerBytes) == "true" {
continue
}
// Compare version IDs chronologically (our version IDs start with timestamp)
if latestVersionId == "" || versionId > latestVersionId {
glog.V(1).Infof("updateLatestVersionAfterDeletion: found newer version %s (file: %s)", versionId, entry.Name)
latestVersionId = versionId
latestVersionFileName = entry.Name
} else {
glog.V(1).Infof("updateLatestVersionAfterDeletion: skipping older version %s", versionId)
}
}
// Update the .versions directory metadata
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
if err != nil {
return fmt.Errorf("failed to get .versions directory: %v", err)
}
if versionsEntry.Extended == nil {
versionsEntry.Extended = make(map[string][]byte)
}
if latestVersionId != "" {
// Update metadata to point to new latest version
versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey] = []byte(latestVersionId)
versionsEntry.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte(latestVersionFileName)
glog.V(2).Infof("updateLatestVersionAfterDeletion: new latest version for %s/%s is %s", bucket, object, latestVersionId)
} else {
// No versions left, remove latest version metadata
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionIdKey)
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionFileNameKey)
glog.V(2).Infof("updateLatestVersionAfterDeletion: no versions left for %s/%s", bucket, object)
}
// Update the .versions directory entry
err = s3a.mkFile(bucketDir, versionsObjectPath, versionsEntry.Chunks, func(updatedEntry *filer_pb.Entry) {
updatedEntry.Extended = versionsEntry.Extended
updatedEntry.Attributes = versionsEntry.Attributes
updatedEntry.Chunks = versionsEntry.Chunks
})
if err != nil {
return fmt.Errorf("failed to update .versions directory metadata: %v", err)
}
return nil
}
@@ -731,8 +419,7 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
// Parse query parameters
query := r.URL.Query()
originalPrefix := query.Get("prefix") // Keep original prefix for response
prefix := originalPrefix // Use for internal processing
prefix := query.Get("prefix")
if prefix != "" && !strings.HasPrefix(prefix, "/") {
prefix = "/" + prefix
}
@@ -757,9 +444,6 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
return
}
// Set the original prefix in the response (not the normalized internal prefix)
result.Prefix = originalPrefix
writeSuccessResponseXML(w, r, result)
}
@@ -771,50 +455,19 @@ func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb
// Get the .versions directory entry to read latest version metadata
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
if err != nil {
// .versions directory doesn't exist - this can happen for objects that existed
// before versioning was enabled on the bucket. Fall back to checking for a
// regular (non-versioned) object file.
glog.V(2).Infof("getLatestObjectVersion: no .versions directory for %s%s, checking for pre-versioning object", bucket, object)
regularEntry, regularErr := s3a.getEntry(bucketDir, object)
if regularErr != nil {
return nil, fmt.Errorf("failed to get %s%s .versions directory and no regular object found: %w", bucket, object, err)
}
glog.V(2).Infof("getLatestObjectVersion: found pre-versioning object for %s/%s", bucket, object)
return regularEntry, nil
return nil, fmt.Errorf("failed to get .versions directory: %w", err)
}
// Check if directory has latest version metadata
if versionsEntry.Extended == nil {
// No metadata means all versioned objects have been deleted.
// Fall back to checking for a pre-versioning object.
glog.V(2).Infof("getLatestObjectVersion: no Extended metadata in .versions directory for %s%s, checking for pre-versioning object", bucket, object)
regularEntry, regularErr := s3a.getEntry(bucketDir, object)
if regularErr != nil {
return nil, fmt.Errorf("no version metadata in .versions directory and no regular object found for %s%s", bucket, object)
}
glog.V(2).Infof("getLatestObjectVersion: found pre-versioning object for %s%s (no Extended metadata case)", bucket, object)
return regularEntry, nil
return nil, fmt.Errorf("no version metadata found in .versions directory for %s/%s", bucket, object)
}
latestVersionIdBytes, hasLatestVersionId := versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey]
latestVersionFileBytes, hasLatestVersionFile := versionsEntry.Extended[s3_constants.ExtLatestVersionFileNameKey]
if !hasLatestVersionId || !hasLatestVersionFile {
// No version metadata means all versioned objects have been deleted.
// Fall back to checking for a pre-versioning object.
glog.V(2).Infof("getLatestObjectVersion: no version metadata in .versions directory for %s/%s, checking for pre-versioning object", bucket, object)
regularEntry, regularErr := s3a.getEntry(bucketDir, object)
if regularErr != nil {
return nil, fmt.Errorf("no version metadata in .versions directory and no regular object found for %s%s", bucket, object)
}
glog.V(2).Infof("getLatestObjectVersion: found pre-versioning object for %s%s after version deletion", bucket, object)
return regularEntry, nil
return nil, fmt.Errorf("incomplete latest version metadata in .versions directory for %s/%s", bucket, object)
}
latestVersionId := string(latestVersionIdBytes)
@@ -831,55 +484,3 @@ func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb
return latestVersionEntry, nil
}
// getObjectOwnerFromVersion extracts object owner information from version entry metadata
func (s3a *S3ApiServer) getObjectOwnerFromVersion(version *ObjectVersion, bucket, objectKey string) CanonicalUser {
// First try to get owner from the version entry itself
if version.Entry != nil && version.Entry.Extended != nil {
if ownerBytes, exists := version.Entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
ownerId := string(ownerBytes)
ownerDisplayName := s3a.iam.GetAccountNameById(ownerId)
return CanonicalUser{ID: ownerId, DisplayName: ownerDisplayName}
}
}
// Fallback: try to get owner from the current version of the object
// This handles cases where older versions might not have owner metadata
if version.VersionId == "null" {
// For null version, check the regular object file
bucketDir := s3a.option.BucketsPath + "/" + bucket
if entry, err := s3a.getEntry(bucketDir, objectKey); err == nil && entry.Extended != nil {
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
ownerId := string(ownerBytes)
ownerDisplayName := s3a.iam.GetAccountNameById(ownerId)
return CanonicalUser{ID: ownerId, DisplayName: ownerDisplayName}
}
}
} else {
// For versioned objects, try to get from latest version metadata
if latestVersion, err := s3a.getLatestObjectVersion(bucket, objectKey); err == nil && latestVersion.Extended != nil {
if ownerBytes, exists := latestVersion.Extended[s3_constants.ExtAmzOwnerKey]; exists {
ownerId := string(ownerBytes)
ownerDisplayName := s3a.iam.GetAccountNameById(ownerId)
return CanonicalUser{ID: ownerId, DisplayName: ownerDisplayName}
}
}
}
// Ultimate fallback: return anonymous if no owner found
return CanonicalUser{ID: s3_constants.AccountAnonymousId, DisplayName: "anonymous"}
}
// getObjectOwnerFromEntry extracts object owner information from a file entry
func (s3a *S3ApiServer) getObjectOwnerFromEntry(entry *filer_pb.Entry) CanonicalUser {
if entry != nil && entry.Extended != nil {
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
ownerId := string(ownerBytes)
ownerDisplayName := s3a.iam.GetAccountNameById(ownerId)
return CanonicalUser{ID: ownerId, DisplayName: ownerDisplayName}
}
}
// Fallback: return anonymous if no owner found
return CanonicalUser{ID: s3_constants.AccountAnonymousId, DisplayName: "anonymous"}
}
-1
View File
@@ -8,7 +8,6 @@ import (
type FullPath string
func NewFullPath(dir, name string) FullPath {
name = strings.TrimSuffix(name, "/")
return FullPath(dir).Child(name)
}
+1 -1
View File
@@ -8,7 +8,7 @@ import (
var (
MAJOR_VERSION = int32(3)
MINOR_VERSION = int32(95)
MINOR_VERSION = int32(94)
VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION)
VERSION = util.SizeLimit + " " + VERSION_NUMBER
COMMIT = ""