mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-05 14:09:24 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7d12a1c04 | ||
|
|
11fb65571f | ||
|
|
6c2a11f1d6 | ||
|
|
851011a32b | ||
|
|
a9a9c732b9 | ||
|
|
6328141019 | ||
|
|
b508e00de7 | ||
|
|
e486817499 | ||
|
|
aec8542b6e | ||
|
|
8fb9df72ae | ||
|
|
af25f92e36 | ||
|
|
66829fa7c8 | ||
|
|
5c24bfc795 | ||
|
|
1d1c717493 | ||
|
|
7661449f82 | ||
|
|
a5a6b5a4d4 | ||
|
|
8121fdeec3 | ||
|
|
f63f40a3d4 | ||
|
|
2d6532d80e | ||
|
|
dce0672ae0 | ||
|
|
40d9f4fa9c | ||
|
|
8cc99ca121 | ||
|
|
c5628b000b | ||
|
|
f419271299 | ||
|
|
a0ab227e07 | ||
|
|
2ad4a924be | ||
|
|
1e1806c7a1 | ||
|
|
eaaef569ce | ||
|
|
1807f4e94d | ||
|
|
7203c78e4d | ||
|
|
1bc503530a | ||
|
|
9f88fd2ea5 | ||
|
|
0123abe49b | ||
|
|
4b92f03b65 | ||
|
|
1e7a3c09d4 | ||
|
|
ddb2105f00 | ||
|
|
bbcd19f6c6 | ||
|
|
fc2fcae543 | ||
|
|
4801f641f1 | ||
|
|
770888df81 | ||
|
|
1dfdd3c84d | ||
|
|
b63855560a | ||
|
|
9f8e0257b0 | ||
|
|
e53cd29eb2 | ||
|
|
eeeb595346 | ||
|
|
678f962ded | ||
|
|
a906358c59 | ||
|
|
12f34f965f | ||
|
|
1db149ecc2 | ||
|
|
59f8444a90 | ||
|
|
bcc7a74a25 | ||
|
|
0d4eeb67be | ||
|
|
85e34df214 | ||
|
|
bb05e6cade | ||
|
|
bc46214445 | ||
|
|
121f6ef2c0 | ||
|
|
22673fee76 | ||
|
|
3fbab20b83 | ||
|
|
b8cbe9b6b2 | ||
|
|
8bb47f3c03 | ||
|
|
7bec2ef94b | ||
|
|
f9cbed7ee1 |
+29
-607
@@ -14,10 +14,10 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
basic-s3-tests:
|
||||
name: Basic S3 tests (KV store)
|
||||
s3tests:
|
||||
name: Ceph S3 tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
@@ -41,7 +41,7 @@ jobs:
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Run Basic S3 tests
|
||||
- name: Run Ceph S3 tests with KV store
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
S3TEST_CONF: ../docker/compose/s3tests.conf
|
||||
@@ -55,76 +55,13 @@ jobs:
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 \
|
||||
-master.port=9333 -volume.port=8080 -filer.port=8888 -s3.port=8000 -metricsPort=9324 \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
|
||||
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
|
||||
pid=$!
|
||||
|
||||
# Wait for all SeaweedFS components to be ready
|
||||
echo "Waiting for SeaweedFS components to start..."
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:9333/cluster/status > /dev/null 2>&1; then
|
||||
echo "Master server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for master server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8080/status > /dev/null 2>&1; then
|
||||
echo "Volume server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for volume server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8888/ > /dev/null 2>&1; then
|
||||
echo "Filer is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for filer... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8000/ > /dev/null 2>&1; then
|
||||
echo "S3 server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "All SeaweedFS components are ready!"
|
||||
sleep 10
|
||||
cd ../s3-tests
|
||||
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
|
||||
|
||||
# Debug: Show the config file contents
|
||||
echo "=== S3 Config File Contents ==="
|
||||
cat ../docker/compose/s3tests.conf
|
||||
echo "=== End Config ==="
|
||||
|
||||
# Additional wait for S3-Filer integration to be fully ready
|
||||
echo "Waiting additional 10 seconds for S3-Filer integration..."
|
||||
sleep 10
|
||||
|
||||
# Test S3 connection before running tests
|
||||
echo "Testing S3 connection..."
|
||||
for i in {1..10}; do
|
||||
if curl -s -f http://localhost:8000/ > /dev/null 2>&1; then
|
||||
echo "S3 connection test successful"
|
||||
break
|
||||
fi
|
||||
echo "S3 connection test failed, retrying... ($i/10)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "✅ S3 server is responding, starting tests..."
|
||||
|
||||
tox -- \
|
||||
s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
|
||||
@@ -283,265 +220,33 @@ jobs:
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
versioning-tests:
|
||||
name: S3 Versioning & Object Lock tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v5.5.0
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Run S3 Object Lock, Retention, and Versioning tests
|
||||
- name: Run S3 Object Lock and Retention tests
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
S3TEST_CONF: ../docker/compose/s3tests.conf
|
||||
shell: bash
|
||||
run: |
|
||||
cd weed
|
||||
go install -buildvcs=false
|
||||
set -x
|
||||
# Create clean data directory for this test run
|
||||
export WEED_DATA_DIR="/tmp/seaweedfs-objectlock-versioning-$(date +%s)"
|
||||
export WEED_DATA_DIR="/tmp/seaweedfs-objectlock-$(date +%s)"
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 \
|
||||
-master.port=9334 -volume.port=8081 -filer.port=8889 -s3.port=8001 -metricsPort=9325 \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
|
||||
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
|
||||
pid=$!
|
||||
|
||||
# Wait for all SeaweedFS components to be ready
|
||||
echo "Waiting for SeaweedFS components to start..."
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:9334/cluster/status > /dev/null 2>&1; then
|
||||
echo "Master server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for master server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8081/status > /dev/null 2>&1; then
|
||||
echo "Volume server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for volume server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8889/ > /dev/null 2>&1; then
|
||||
echo "Filer is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for filer... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8001/ > /dev/null 2>&1; then
|
||||
echo "S3 server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "All SeaweedFS components are ready!"
|
||||
sleep 10
|
||||
cd ../s3-tests
|
||||
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
|
||||
# Fix bucket creation conflicts in versioning tests by replacing _create_objects calls
|
||||
sed -i 's/bucket_name = _create_objects(bucket_name=bucket_name,keys=key_names)/# Use the existing bucket for object creation\n client = get_client()\n for key in key_names:\n client.put_object(Bucket=bucket_name, Body=key, Key=key)/' s3tests_boto3/functional/test_s3.py
|
||||
sed -i 's/bucket = _create_objects(bucket_name=bucket_name, keys=key_names)/# Use the existing bucket for object creation\n client = get_client()\n for key in key_names:\n client.put_object(Bucket=bucket_name, Body=key, Key=key)/' s3tests_boto3/functional/test_s3.py
|
||||
# Create and update s3tests.conf to use port 8001
|
||||
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-versioning.conf
|
||||
sed -i 's/port = 8000/port = 8001/g' ../docker/compose/s3tests-versioning.conf
|
||||
sed -i 's/:8000/:8001/g' ../docker/compose/s3tests-versioning.conf
|
||||
sed -i 's/localhost:8000/localhost:8001/g' ../docker/compose/s3tests-versioning.conf
|
||||
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8001/g' ../docker/compose/s3tests-versioning.conf
|
||||
export S3TEST_CONF=../docker/compose/s3tests-versioning.conf
|
||||
|
||||
# Debug: Show the config file contents
|
||||
echo "=== S3 Config File Contents ==="
|
||||
cat ../docker/compose/s3tests-versioning.conf
|
||||
echo "=== End Config ==="
|
||||
|
||||
# Additional wait for S3-Filer integration to be fully ready
|
||||
echo "Waiting additional 10 seconds for S3-Filer integration..."
|
||||
sleep 10
|
||||
|
||||
# Test S3 connection before running tests
|
||||
echo "Testing S3 connection..."
|
||||
for i in {1..10}; do
|
||||
if curl -s -f http://localhost:8001/ > /dev/null 2>&1; then
|
||||
echo "S3 connection test successful"
|
||||
break
|
||||
fi
|
||||
echo "S3 connection test failed, retrying... ($i/10)"
|
||||
sleep 2
|
||||
done
|
||||
# tox -- s3tests_boto3/functional/test_s3.py -k "object_lock or (versioning and not test_versioning_obj_suspend_versions and not test_bucket_list_return_data_versioning and not test_versioning_concurrent_multi_object_delete)" --tb=short
|
||||
tox -- s3tests_boto3/functional/test_s3.py -k "object_lock or versioning" --tb=short
|
||||
# Run object lock tests by pattern matching test names
|
||||
tox -- -k "object_lock" --tb=short
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
cors-tests:
|
||||
name: S3 CORS tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v5.5.0
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Run S3 CORS tests
|
||||
timeout-minutes: 10
|
||||
shell: bash
|
||||
run: |
|
||||
cd weed
|
||||
go install -buildvcs=false
|
||||
set -x
|
||||
# Create clean data directory for this test run
|
||||
export WEED_DATA_DIR="/tmp/seaweedfs-cors-test-$(date +%s)"
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 \
|
||||
-master.port=9335 -volume.port=8082 -filer.port=8890 -s3.port=8002 -metricsPort=9326 \
|
||||
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
|
||||
pid=$!
|
||||
|
||||
# Wait for all SeaweedFS components to be ready
|
||||
echo "Waiting for SeaweedFS components to start..."
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:9335/cluster/status > /dev/null 2>&1; then
|
||||
echo "Master server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for master server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8082/status > /dev/null 2>&1; then
|
||||
echo "Volume server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for volume server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8890/ > /dev/null 2>&1; then
|
||||
echo "Filer is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for filer... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8002/ > /dev/null 2>&1; then
|
||||
echo "S3 server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "All SeaweedFS components are ready!"
|
||||
cd ../s3-tests
|
||||
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
|
||||
# Create and update s3tests.conf to use port 8002
|
||||
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-cors.conf
|
||||
sed -i 's/port = 8000/port = 8002/g' ../docker/compose/s3tests-cors.conf
|
||||
sed -i 's/:8000/:8002/g' ../docker/compose/s3tests-cors.conf
|
||||
sed -i 's/localhost:8000/localhost:8002/g' ../docker/compose/s3tests-cors.conf
|
||||
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8002/g' ../docker/compose/s3tests-cors.conf
|
||||
export S3TEST_CONF=../docker/compose/s3tests-cors.conf
|
||||
|
||||
# Debug: Show the config file contents
|
||||
echo "=== S3 Config File Contents ==="
|
||||
cat ../docker/compose/s3tests-cors.conf
|
||||
echo "=== End Config ==="
|
||||
|
||||
# Additional wait for S3-Filer integration to be fully ready
|
||||
echo "Waiting additional 10 seconds for S3-Filer integration..."
|
||||
sleep 10
|
||||
|
||||
# Test S3 connection before running tests
|
||||
echo "Testing S3 connection..."
|
||||
for i in {1..10}; do
|
||||
if curl -s -f http://localhost:8002/ > /dev/null 2>&1; then
|
||||
echo "S3 connection test successful"
|
||||
break
|
||||
fi
|
||||
echo "S3 connection test failed, retrying... ($i/10)"
|
||||
sleep 2
|
||||
done
|
||||
# Run CORS-specific tests from s3-tests suite
|
||||
tox -- s3tests_boto3/functional/test_s3.py -k "cors" --tb=short || echo "No CORS tests found in s3-tests suite"
|
||||
# If no specific CORS tests exist, run bucket configuration tests that include CORS
|
||||
tox -- s3tests_boto3/functional/test_s3.py::test_put_bucket_cors || echo "No put_bucket_cors test found"
|
||||
tox -- s3tests_boto3/functional/test_s3.py::test_get_bucket_cors || echo "No get_bucket_cors test found"
|
||||
tox -- s3tests_boto3/functional/test_s3.py::test_delete_bucket_cors || echo "No delete_bucket_cors test found"
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
copy-tests:
|
||||
name: SeaweedFS Custom S3 Copy tests
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v5.5.0
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Run SeaweedFS Custom S3 Copy tests
|
||||
timeout-minutes: 10
|
||||
shell: bash
|
||||
@@ -554,322 +259,39 @@ jobs:
|
||||
set -x
|
||||
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 \
|
||||
-master.port=9336 -volume.port=8083 -filer.port=8891 -s3.port=8003 -metricsPort=9327 \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
|
||||
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
|
||||
pid=$!
|
||||
|
||||
# Wait for all SeaweedFS components to be ready
|
||||
echo "Waiting for SeaweedFS components to start..."
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:9336/cluster/status > /dev/null 2>&1; then
|
||||
echo "Master server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for master server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8083/status > /dev/null 2>&1; then
|
||||
echo "Volume server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for volume server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8891/ > /dev/null 2>&1; then
|
||||
echo "Filer is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for filer... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8003/ > /dev/null 2>&1; then
|
||||
echo "S3 server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 server... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "All SeaweedFS components are ready!"
|
||||
cd ../test/s3/copying
|
||||
# Patch Go tests to use the correct S3 endpoint (port 8003)
|
||||
sed -i 's/http:\/\/127\.0\.0\.1:8000/http:\/\/127.0.0.1:8003/g' s3_copying_test.go
|
||||
|
||||
# Debug: Show what endpoint the Go tests will use
|
||||
echo "=== Go Test Configuration ==="
|
||||
grep -n "127.0.0.1" s3_copying_test.go || echo "No IP configuration found"
|
||||
echo "=== End Configuration ==="
|
||||
|
||||
# Additional wait for S3-Filer integration to be fully ready
|
||||
echo "Waiting additional 10 seconds for S3-Filer integration..."
|
||||
sleep 10
|
||||
|
||||
# Test S3 connection before running tests
|
||||
echo "Testing S3 connection..."
|
||||
for i in {1..10}; do
|
||||
if curl -s -f http://localhost:8003/ > /dev/null 2>&1; then
|
||||
echo "S3 connection test successful"
|
||||
break
|
||||
fi
|
||||
echo "S3 connection test failed, retrying... ($i/10)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
cd ../test/s3/copying
|
||||
go test -v
|
||||
kill -9 $pid || true
|
||||
# Clean up data directory
|
||||
rm -rf "$WEED_DATA_DIR" || true
|
||||
|
||||
sql-store-tests:
|
||||
name: Basic S3 tests (SQL store)
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@v5.5.0
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
id: go
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Clone s3-tests
|
||||
run: |
|
||||
git clone https://github.com/ceph/s3-tests.git
|
||||
cd s3-tests
|
||||
pip install -r requirements.txt
|
||||
pip install tox
|
||||
pip install -e .
|
||||
|
||||
- name: Run Ceph S3 tests with SQL store
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
S3TEST_CONF: ../docker/compose/s3tests.conf
|
||||
shell: bash
|
||||
run: |
|
||||
cd weed
|
||||
|
||||
# Debug: Check for port conflicts before starting
|
||||
echo "=== Pre-start Port Check ==="
|
||||
netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" || echo "Ports are free"
|
||||
|
||||
# Kill any existing weed processes that might interfere
|
||||
echo "=== Cleanup existing processes ==="
|
||||
pkill -f weed || echo "No weed processes found"
|
||||
|
||||
# More aggressive port cleanup using multiple methods
|
||||
for port in 9337 8085 8892 8004 9328; do
|
||||
echo "Cleaning port $port..."
|
||||
|
||||
# Method 1: lsof
|
||||
pid=$(lsof -ti :$port 2>/dev/null || echo "")
|
||||
if [ -n "$pid" ]; then
|
||||
echo "Found process $pid using port $port (via lsof)"
|
||||
kill -9 $pid 2>/dev/null || echo "Failed to kill $pid"
|
||||
fi
|
||||
|
||||
# Method 2: netstat + ps (for cases where lsof fails)
|
||||
netstat_pids=$(netstat -tlnp 2>/dev/null | grep ":$port " | awk '{print $7}' | cut -d'/' -f1 | grep -v '^-$' || echo "")
|
||||
for npid in $netstat_pids; do
|
||||
if [ -n "$npid" ] && [ "$npid" != "-" ]; then
|
||||
echo "Found process $npid using port $port (via netstat)"
|
||||
kill -9 $npid 2>/dev/null || echo "Failed to kill $npid"
|
||||
fi
|
||||
done
|
||||
|
||||
# Method 3: fuser (if available)
|
||||
if command -v fuser >/dev/null 2>&1; then
|
||||
fuser -k ${port}/tcp 2>/dev/null || echo "No process found via fuser for port $port"
|
||||
fi
|
||||
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Wait for ports to be released
|
||||
sleep 5
|
||||
|
||||
echo "=== Post-cleanup Port Check ==="
|
||||
netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" || echo "All ports are now free"
|
||||
|
||||
# If any ports are still in use, fail fast
|
||||
if netstat -tulpn | grep -E "(9337|8085|8892|8004|9328)" >/dev/null 2>&1; then
|
||||
echo "❌ ERROR: Some ports are still in use after aggressive cleanup!"
|
||||
echo "=== Detailed Port Analysis ==="
|
||||
for port in 9337 8085 8892 8004 9328; do
|
||||
echo "Port $port:"
|
||||
netstat -tlnp 2>/dev/null | grep ":$port " || echo " Not in use"
|
||||
lsof -i :$port 2>/dev/null || echo " No lsof info"
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
|
||||
go install -tags "sqlite" -buildvcs=false
|
||||
# Create clean data directory for this test run with unique timestamp and process ID
|
||||
export WEED_DATA_DIR="/tmp/seaweedfs-sql-test-$(date +%s)-$$"
|
||||
# Create clean data directory for this test run
|
||||
export WEED_DATA_DIR="/tmp/seaweedfs-sql-test-$(date +%s)"
|
||||
mkdir -p "$WEED_DATA_DIR"
|
||||
chmod 777 "$WEED_DATA_DIR"
|
||||
|
||||
# SQLite-specific configuration
|
||||
export WEED_LEVELDB2_ENABLED="false"
|
||||
export WEED_SQLITE_ENABLED="true"
|
||||
export WEED_SQLITE_DBFILE="$WEED_DATA_DIR/filer.db"
|
||||
|
||||
echo "=== SQL Store Configuration ==="
|
||||
echo "Data Dir: $WEED_DATA_DIR"
|
||||
echo "SQLite DB: $WEED_SQLITE_DBFILE"
|
||||
echo "LEVELDB2_ENABLED: $WEED_LEVELDB2_ENABLED"
|
||||
echo "SQLITE_ENABLED: $WEED_SQLITE_ENABLED"
|
||||
|
||||
export WEED_LEVELDB2_ENABLED="false" WEED_SQLITE_ENABLED="true" WEED_SQLITE_DBFILE="$WEED_DATA_DIR/filer.db"
|
||||
set -x
|
||||
weed -v 1 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
||||
weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
|
||||
-dir="$WEED_DATA_DIR" \
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=100 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 \
|
||||
-master.port=9337 -volume.port=8085 -filer.port=8892 -s3.port=8004 -metricsPort=9328 \
|
||||
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json \
|
||||
> /tmp/seaweedfs-sql-server.log 2>&1 &
|
||||
-master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
|
||||
-volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
|
||||
-s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
|
||||
pid=$!
|
||||
|
||||
echo "=== Server started with PID: $pid ==="
|
||||
|
||||
# Wait for all SeaweedFS components to be ready
|
||||
echo "Waiting for SeaweedFS components to start..."
|
||||
|
||||
# Check if server process is still alive before waiting
|
||||
if ! kill -0 $pid 2>/dev/null; then
|
||||
echo "❌ Server process died immediately after start"
|
||||
echo "=== Immediate Log Check ==="
|
||||
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null || echo "No log available"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sleep 5 # Give SQLite more time to initialize
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:9337/cluster/status > /dev/null 2>&1; then
|
||||
echo "Master server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for master server... ($i/30)"
|
||||
# Check if server process is still alive
|
||||
if ! kill -0 $pid 2>/dev/null; then
|
||||
echo "❌ Server process died while waiting for master"
|
||||
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8085/status > /dev/null 2>&1; then
|
||||
echo "Volume server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for volume server... ($i/30)"
|
||||
if ! kill -0 $pid 2>/dev/null; then
|
||||
echo "❌ Server process died while waiting for volume"
|
||||
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8892/ > /dev/null 2>&1; then
|
||||
echo "Filer (SQLite) is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for filer (SQLite)... ($i/30)"
|
||||
if ! kill -0 $pid 2>/dev/null; then
|
||||
echo "❌ Server process died while waiting for filer"
|
||||
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Extra wait for SQLite filer to fully initialize
|
||||
echo "Giving SQLite filer extra time to initialize..."
|
||||
sleep 5
|
||||
|
||||
for i in {1..30}; do
|
||||
if curl -s http://localhost:8004/ > /dev/null 2>&1; then
|
||||
echo "S3 server is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting for S3 server... ($i/30)"
|
||||
if ! kill -0 $pid 2>/dev/null; then
|
||||
echo "❌ Server process died while waiting for S3"
|
||||
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "All SeaweedFS components are ready!"
|
||||
sleep 10
|
||||
cd ../s3-tests
|
||||
sed -i "s/assert prefixes == \['foo%2B1\/', 'foo\/', 'quux%20ab\/'\]/assert prefixes == \['foo\/', 'foo%2B1\/', 'quux%20ab\/'\]/" s3tests_boto3/functional/test_s3.py
|
||||
# Create and update s3tests.conf to use port 8004
|
||||
cp ../docker/compose/s3tests.conf ../docker/compose/s3tests-sql.conf
|
||||
sed -i 's/port = 8000/port = 8004/g' ../docker/compose/s3tests-sql.conf
|
||||
sed -i 's/:8000/:8004/g' ../docker/compose/s3tests-sql.conf
|
||||
sed -i 's/localhost:8000/localhost:8004/g' ../docker/compose/s3tests-sql.conf
|
||||
sed -i 's/127\.0\.0\.1:8000/127.0.0.1:8004/g' ../docker/compose/s3tests-sql.conf
|
||||
export S3TEST_CONF=../docker/compose/s3tests-sql.conf
|
||||
|
||||
# Debug: Show the config file contents
|
||||
echo "=== S3 Config File Contents ==="
|
||||
cat ../docker/compose/s3tests-sql.conf
|
||||
echo "=== End Config ==="
|
||||
|
||||
# Additional wait for S3-Filer integration to be fully ready
|
||||
echo "Waiting additional 10 seconds for S3-Filer integration..."
|
||||
sleep 10
|
||||
|
||||
# Test S3 connection before running tests
|
||||
echo "Testing S3 connection..."
|
||||
|
||||
# Debug: Check if SeaweedFS processes are running
|
||||
echo "=== Process Status ==="
|
||||
ps aux | grep -E "(weed|seaweedfs)" | grep -v grep || echo "No SeaweedFS processes found"
|
||||
|
||||
# Debug: Check port status
|
||||
echo "=== Port Status ==="
|
||||
netstat -tulpn | grep -E "(8004|9337|8085|8892)" || echo "Ports not found"
|
||||
|
||||
# Debug: Check server logs
|
||||
echo "=== Recent Server Logs ==="
|
||||
echo "--- SQL Server Log ---"
|
||||
tail -20 /tmp/seaweedfs-sql-server.log 2>/dev/null || echo "No SQL server log found"
|
||||
echo "--- Other Logs ---"
|
||||
ls -la /tmp/seaweedfs-*.log 2>/dev/null || echo "No other log files found"
|
||||
|
||||
for i in {1..10}; do
|
||||
if curl -s -f http://localhost:8004/ > /dev/null 2>&1; then
|
||||
echo "S3 connection test successful"
|
||||
break
|
||||
fi
|
||||
echo "S3 connection test failed, retrying... ($i/10)"
|
||||
|
||||
# Debug: Try different HTTP methods
|
||||
echo "Debug: Testing different endpoints..."
|
||||
curl -s -I http://localhost:8004/ || echo "HEAD request failed"
|
||||
curl -s http://localhost:8004/status || echo "Status endpoint failed"
|
||||
|
||||
sleep 2
|
||||
done
|
||||
tox -- \
|
||||
s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
|
||||
s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
|
||||
|
||||
@@ -110,5 +110,3 @@ test/s3/cors/cors.test
|
||||
/test/s3/retention/filerldb2
|
||||
test/s3/retention/weed-server.pid
|
||||
test/s3/retention/weed-test.log
|
||||
/test/s3/versioning/test-volume-data
|
||||
test/s3/versioning/weed-test.log
|
||||
|
||||
@@ -23,7 +23,7 @@ server: install
|
||||
benchmark: install warp_install
|
||||
pkill weed || true
|
||||
pkill warp || true
|
||||
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
|
||||
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
|
||||
warp client &
|
||||
while ! nc -z localhost 8000 ; do sleep 1 ; done
|
||||
warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm
|
||||
|
||||
@@ -10,7 +10,7 @@ services:
|
||||
- 18084:18080
|
||||
- 8888:8888
|
||||
- 18888:18888
|
||||
command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
|
||||
command: "server -ip=server1 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
|
||||
volumes:
|
||||
- ./master-cloud.toml:/etc/seaweedfs/master.toml
|
||||
depends_on:
|
||||
@@ -25,4 +25,4 @@ services:
|
||||
- 8889:8888
|
||||
- 18889:18888
|
||||
- 8334:8333
|
||||
command: "server -ip=server2 -filer -s3 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
|
||||
command: "server -ip=server2 -filer -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
|
||||
|
||||
@@ -3,7 +3,7 @@ version: '3.9'
|
||||
services:
|
||||
server-left:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
|
||||
command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
healthcheck:
|
||||
@@ -13,7 +13,7 @@ services:
|
||||
timeout: 30s
|
||||
server-right:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1"
|
||||
command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
healthcheck:
|
||||
|
||||
@@ -6,7 +6,7 @@ services:
|
||||
ports:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
command: "master -ip=master -volumeSizeLimitMB=100"
|
||||
command: "master -ip=master -volumeSizeLimitMB=1024"
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
|
||||
@@ -6,7 +6,7 @@ services:
|
||||
ports:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
command: "master -ip=master -volumeSizeLimitMB=100"
|
||||
command: "master -ip=master -volumeSizeLimitMB=1024"
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
|
||||
@@ -11,7 +11,7 @@ services:
|
||||
ports:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
command: "master -ip=master -volumeSizeLimitMB=100"
|
||||
command: "master -ip=master -volumeSizeLimitMB=1024"
|
||||
volume:
|
||||
image: chrislusf/seaweedfs:local
|
||||
ports:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "3.95"
|
||||
appVersion: "3.94"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.0.395
|
||||
version: 4.0.394
|
||||
|
||||
@@ -222,13 +222,13 @@ test-with-server: start-server
|
||||
test-versioning-with-configs: check-deps
|
||||
@echo "Testing with different S3 configurations..."
|
||||
@echo "Testing with empty folder allowed..."
|
||||
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowEmptyFolder=true -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
|
||||
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowEmptyFolder=true -filer -master.volumeSizeLimitMB=1024 -volume.max=100 > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid
|
||||
@sleep 5
|
||||
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
|
||||
@if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi
|
||||
@sleep 2
|
||||
@echo "Testing with delete bucket not empty disabled..."
|
||||
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=100 -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
|
||||
@$(WEED_BINARY) server -s3 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false -filer -master.volumeSizeLimitMB=1024 -volume.max=100 > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid
|
||||
@sleep 5
|
||||
@go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true
|
||||
@if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -164,16 +164,6 @@ func checkVersioningStatus(t *testing.T, client *s3.Client, bucketName string, e
|
||||
assert.Equal(t, expectedStatus, resp.Status)
|
||||
}
|
||||
|
||||
// checkVersioningStatusEmpty verifies that a bucket has no versioning configuration (newly created bucket)
|
||||
func checkVersioningStatusEmpty(t *testing.T, client *s3.Client, bucketName string) {
|
||||
resp, err := client.GetBucketVersioning(context.TODO(), &s3.GetBucketVersioningInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
// AWS S3 returns an empty versioning configuration (no Status field) for buckets that have never had versioning configured, such as newly created buckets.
|
||||
assert.Empty(t, resp.Status, "Newly created bucket should have empty versioning status")
|
||||
}
|
||||
|
||||
// putObject puts an object into a bucket
|
||||
func putObject(t *testing.T, client *s3.Client, bucketName, key, content string) *s3.PutObjectOutput {
|
||||
resp, err := client.PutObject(context.TODO(), &s3.PutObjectInput{
|
||||
@@ -294,9 +284,8 @@ func TestVersioningBasicWorkflow(t *testing.T) {
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
|
||||
// Initially, versioning should be unset/empty (not suspended) for newly created buckets
|
||||
// This matches AWS S3 behavior where new buckets have no versioning status
|
||||
checkVersioningStatusEmpty(t, client, bucketName)
|
||||
// Initially, versioning should be suspended/disabled
|
||||
checkVersioningStatus(t, client, bucketName, types.BucketVersioningStatusSuspended)
|
||||
|
||||
// Enable versioning
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
Binary file not shown.
+1
-1
@@ -23,7 +23,7 @@ debug_mount:
|
||||
|
||||
debug_server:
|
||||
go build -gcflags="all=-N -l"
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=100 -volume.preStopSeconds=1
|
||||
dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./weed -- server -dir=~/tmp/99 -filer -volume.port=8343 -s3 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1
|
||||
|
||||
debug_volume:
|
||||
go build -tags=5BytesOffset -gcflags="all=-N -l"
|
||||
|
||||
@@ -138,8 +138,9 @@ func (r *LockRing) compactSnapshots() {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
|
||||
// Always attempt compaction when called, regardless of lastCompactTime
|
||||
// This ensures proper cleanup even with multiple concurrent compaction requests
|
||||
if r.lastCompactTime.After(r.lastUpdateTime) {
|
||||
return
|
||||
}
|
||||
|
||||
ts := time.Now()
|
||||
// remove old snapshots
|
||||
|
||||
@@ -51,13 +51,6 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
entry.Extended["key"] = []byte(*input.Key)
|
||||
|
||||
// Set object owner for multipart upload
|
||||
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
|
||||
if amzAccountId != "" {
|
||||
entry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(amzAccountId)
|
||||
}
|
||||
|
||||
for k, v := range input.Metadata {
|
||||
entry.Extended[k] = []byte(*v)
|
||||
}
|
||||
@@ -99,7 +92,7 @@ type CompleteMultipartUploadResult struct {
|
||||
VersionId *string `xml:"-"`
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) {
|
||||
func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) {
|
||||
|
||||
glog.V(2).Infof("completeMultipartUpload input %v", input)
|
||||
if len(parts.Parts) == 0 {
|
||||
@@ -245,10 +238,32 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
}
|
||||
|
||||
entryName, dirName := s3a.getEntryNameAndDir(input)
|
||||
err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) {
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
entry.Extended[s3_constants.SeaweedFSUploadId] = []byte(*input.UploadId)
|
||||
for k, v := range pentry.Extended {
|
||||
if k != "key" {
|
||||
entry.Extended[k] = v
|
||||
}
|
||||
}
|
||||
if pentry.Attributes.Mime != "" {
|
||||
entry.Attributes.Mime = pentry.Attributes.Mime
|
||||
} else if mime != "" {
|
||||
entry.Attributes.Mime = mime
|
||||
}
|
||||
entry.Attributes.FileSize = uint64(offset)
|
||||
})
|
||||
|
||||
// Check if versioning is configured for this bucket BEFORE creating any files
|
||||
versioningState, vErr := s3a.getVersioningState(*input.Bucket)
|
||||
if vErr == nil && versioningState == s3_constants.VersioningEnabled {
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err)
|
||||
return nil, s3err.ErrInternalError
|
||||
}
|
||||
|
||||
// Check if versioning is enabled for this bucket
|
||||
versioningEnabled, vErr := s3a.isVersioningEnabled(*input.Bucket)
|
||||
if vErr == nil && versioningEnabled {
|
||||
// For versioned buckets, create a version and return the version ID
|
||||
versionId := generateVersionId()
|
||||
versionFileName := s3a.getVersionFileName(versionId)
|
||||
@@ -261,13 +276,6 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
}
|
||||
versionEntry.Extended[s3_constants.ExtVersionIdKey] = []byte(versionId)
|
||||
versionEntry.Extended[s3_constants.SeaweedFSUploadId] = []byte(*input.UploadId)
|
||||
|
||||
// Set object owner for versioned multipart objects
|
||||
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
|
||||
if amzAccountId != "" {
|
||||
versionEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(amzAccountId)
|
||||
}
|
||||
|
||||
for k, v := range pentry.Extended {
|
||||
if k != "key" {
|
||||
versionEntry.Extended[k] = v
|
||||
@@ -293,8 +301,19 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
return nil, s3err.ErrInternalError
|
||||
}
|
||||
|
||||
// For versioned buckets, don't create a main object file - all content is stored in .versions directory
|
||||
// The latest version information is tracked in the .versions directory metadata
|
||||
// Create a delete marker for the main object (latest version)
|
||||
err = s3a.mkFile(dirName, entryName, nil, func(mainEntry *filer_pb.Entry) {
|
||||
if mainEntry.Extended == nil {
|
||||
mainEntry.Extended = make(map[string][]byte)
|
||||
}
|
||||
mainEntry.Extended[s3_constants.ExtVersionIdKey] = []byte(versionId)
|
||||
mainEntry.Extended[s3_constants.ExtDeleteMarkerKey] = []byte("false") // This is the latest version, not a delete marker
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload: failed to update main entry: %v", err)
|
||||
return nil, s3err.ErrInternalError
|
||||
}
|
||||
|
||||
output = &CompleteMultipartUploadResult{
|
||||
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))),
|
||||
@@ -303,78 +322,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
Key: objectKey(input.Key),
|
||||
VersionId: aws.String(versionId),
|
||||
}
|
||||
} else if vErr == nil && versioningState == s3_constants.VersioningSuspended {
|
||||
// For suspended versioning, add "null" version ID metadata and return "null" version ID
|
||||
err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) {
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
entry.Extended[s3_constants.ExtVersionIdKey] = []byte("null")
|
||||
|
||||
// Set object owner for suspended versioning multipart objects
|
||||
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
|
||||
if amzAccountId != "" {
|
||||
entry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(amzAccountId)
|
||||
}
|
||||
|
||||
for k, v := range pentry.Extended {
|
||||
if k != "key" {
|
||||
entry.Extended[k] = v
|
||||
}
|
||||
}
|
||||
if pentry.Attributes.Mime != "" {
|
||||
entry.Attributes.Mime = pentry.Attributes.Mime
|
||||
} else if mime != "" {
|
||||
entry.Attributes.Mime = mime
|
||||
}
|
||||
entry.Attributes.FileSize = uint64(offset)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload: failed to create suspended versioning object: %v", err)
|
||||
return nil, s3err.ErrInternalError
|
||||
}
|
||||
|
||||
// Note: Suspended versioning should NOT return VersionId field according to AWS S3 spec
|
||||
output = &CompleteMultipartUploadResult{
|
||||
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))),
|
||||
Bucket: input.Bucket,
|
||||
ETag: aws.String("\"" + filer.ETagChunks(finalParts) + "\""),
|
||||
Key: objectKey(input.Key),
|
||||
// VersionId field intentionally omitted for suspended versioning
|
||||
}
|
||||
} else {
|
||||
// For non-versioned buckets, create main object file
|
||||
err = s3a.mkFile(dirName, entryName, finalParts, func(entry *filer_pb.Entry) {
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
entry.Extended[s3_constants.SeaweedFSUploadId] = []byte(*input.UploadId)
|
||||
|
||||
// Set object owner for non-versioned multipart objects
|
||||
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
|
||||
if amzAccountId != "" {
|
||||
entry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(amzAccountId)
|
||||
}
|
||||
|
||||
for k, v := range pentry.Extended {
|
||||
if k != "key" {
|
||||
entry.Extended[k] = v
|
||||
}
|
||||
}
|
||||
if pentry.Attributes.Mime != "" {
|
||||
entry.Attributes.Mime = pentry.Attributes.Mime
|
||||
} else if mime != "" {
|
||||
entry.Attributes.Mime = mime
|
||||
}
|
||||
entry.Attributes.FileSize = uint64(offset)
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s/%s error: %v", dirName, entryName, err)
|
||||
return nil, s3err.ErrInternalError
|
||||
}
|
||||
|
||||
// For non-versioned buckets, return response without VersionId
|
||||
output = &CompleteMultipartUploadResult{
|
||||
Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer.ToHttpAddress(), urlEscapeObject(dirName), urlPathEscape(entryName))),
|
||||
|
||||
@@ -2,7 +2,6 @@ package s3api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -98,11 +97,10 @@ func (s3a *S3ApiServer) getBucketConfig(bucket string) (*BucketConfig, s3err.Err
|
||||
return config, s3err.ErrNone
|
||||
}
|
||||
|
||||
// Try to get from filer
|
||||
entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
// Load from filer
|
||||
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
// Bucket doesn't exist
|
||||
if err == filer_pb.ErrNotFound {
|
||||
return nil, s3err.ErrNoSuchBucket
|
||||
}
|
||||
glog.Errorf("getBucketConfig: failed to get bucket entry for %s: %v", bucket, err)
|
||||
@@ -111,25 +109,25 @@ func (s3a *S3ApiServer) getBucketConfig(bucket string) (*BucketConfig, s3err.Err
|
||||
|
||||
config := &BucketConfig{
|
||||
Name: bucket,
|
||||
Entry: entry,
|
||||
Entry: bucketEntry,
|
||||
}
|
||||
|
||||
// Extract configuration from extended attributes
|
||||
if entry.Extended != nil {
|
||||
if versioning, exists := entry.Extended[s3_constants.ExtVersioningKey]; exists {
|
||||
if bucketEntry.Extended != nil {
|
||||
if versioning, exists := bucketEntry.Extended[s3_constants.ExtVersioningKey]; exists {
|
||||
config.Versioning = string(versioning)
|
||||
}
|
||||
if ownership, exists := entry.Extended[s3_constants.ExtOwnershipKey]; exists {
|
||||
if ownership, exists := bucketEntry.Extended[s3_constants.ExtOwnershipKey]; exists {
|
||||
config.Ownership = string(ownership)
|
||||
}
|
||||
if acl, exists := entry.Extended[s3_constants.ExtAmzAclKey]; exists {
|
||||
if acl, exists := bucketEntry.Extended[s3_constants.ExtAmzAclKey]; exists {
|
||||
config.ACL = acl
|
||||
}
|
||||
if owner, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
|
||||
if owner, exists := bucketEntry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
|
||||
config.Owner = string(owner)
|
||||
}
|
||||
// Parse Object Lock configuration if present
|
||||
if objectLockConfig, found := LoadObjectLockConfigurationFromExtended(entry); found {
|
||||
if objectLockConfig, found := LoadObjectLockConfigurationFromExtended(bucketEntry); found {
|
||||
config.ObjectLockConfig = objectLockConfig
|
||||
glog.V(2).Infof("getBucketConfig: cached Object Lock configuration for bucket %s", bucket)
|
||||
}
|
||||
@@ -137,7 +135,7 @@ func (s3a *S3ApiServer) getBucketConfig(bucket string) (*BucketConfig, s3err.Err
|
||||
|
||||
// Load CORS configuration from .s3metadata
|
||||
if corsConfig, err := s3a.loadCORSFromMetadata(bucket); err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
// Missing metadata is not an error; fall back cleanly
|
||||
glog.V(2).Infof("CORS metadata not found for bucket %s, falling back to default behavior", bucket)
|
||||
} else {
|
||||
@@ -221,40 +219,6 @@ func (s3a *S3ApiServer) isVersioningEnabled(bucket string) (bool, error) {
|
||||
return config.Versioning == s3_constants.VersioningEnabled || config.ObjectLockConfig != nil, nil
|
||||
}
|
||||
|
||||
// isVersioningConfigured checks if versioning has been configured (either Enabled or Suspended)
|
||||
func (s3a *S3ApiServer) isVersioningConfigured(bucket string) (bool, error) {
|
||||
config, errCode := s3a.getBucketConfig(bucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
if errCode == s3err.ErrNoSuchBucket {
|
||||
return false, filer_pb.ErrNotFound
|
||||
}
|
||||
return false, fmt.Errorf("failed to get bucket config: %v", errCode)
|
||||
}
|
||||
|
||||
// Versioning is configured if explicitly set to either "Enabled" or "Suspended"
|
||||
// OR if object lock is enabled (which forces versioning)
|
||||
return config.Versioning != "" || config.ObjectLockConfig != nil, nil
|
||||
}
|
||||
|
||||
// getVersioningState returns the detailed versioning state for a bucket
|
||||
func (s3a *S3ApiServer) getVersioningState(bucket string) (string, error) {
|
||||
config, errCode := s3a.getBucketConfig(bucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
if errCode == s3err.ErrNoSuchBucket {
|
||||
return "", filer_pb.ErrNotFound
|
||||
}
|
||||
return "", fmt.Errorf("failed to get bucket config: %v", errCode)
|
||||
}
|
||||
|
||||
// If object lock is enabled, versioning must be enabled regardless of explicit setting
|
||||
if config.ObjectLockConfig != nil {
|
||||
return s3_constants.VersioningEnabled, nil
|
||||
}
|
||||
|
||||
// Return the explicit versioning status (empty string means never configured)
|
||||
return config.Versioning, nil
|
||||
}
|
||||
|
||||
// getBucketVersioningStatus returns the versioning status for a bucket
|
||||
func (s3a *S3ApiServer) getBucketVersioningStatus(bucket string) (string, s3err.ErrorCode) {
|
||||
config, errCode := s3a.getBucketConfig(bucket)
|
||||
@@ -262,8 +226,10 @@ func (s3a *S3ApiServer) getBucketVersioningStatus(bucket string) (string, s3err.
|
||||
return "", errCode
|
||||
}
|
||||
|
||||
// Return exactly what's stored - empty string means versioning was never configured
|
||||
// This matches AWS S3 behavior where new buckets have no Status field in GetBucketVersioning response
|
||||
if config.Versioning == "" {
|
||||
return s3_constants.VersioningSuspended, s3err.ErrNone
|
||||
}
|
||||
|
||||
return config.Versioning, s3err.ErrNone
|
||||
}
|
||||
|
||||
@@ -312,7 +278,7 @@ func (s3a *S3ApiServer) loadCORSFromMetadata(bucket string) (*cors.CORSConfigura
|
||||
entry, err := s3a.getEntry("", bucketMetadataPath)
|
||||
if err != nil {
|
||||
glog.V(3).Infof("loadCORSFromMetadata: error retrieving metadata for bucket %s: %v", bucket, err)
|
||||
return nil, fmt.Errorf("error retrieving CORS metadata for bucket %s: %w", bucket, err)
|
||||
return nil, fmt.Errorf("error retrieving metadata for bucket %s: %v", bucket, err)
|
||||
}
|
||||
if entry == nil {
|
||||
glog.V(3).Infof("loadCORSFromMetadata: no metadata entry found for bucket %s", bucket)
|
||||
|
||||
@@ -230,7 +230,7 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
|
||||
bucket, _ := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("HeadBucketHandler %s", bucket)
|
||||
|
||||
if entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket); entry == nil || errors.Is(err, filer_pb.ErrNotFound) {
|
||||
if entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket); entry == nil || err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
@@ -240,7 +240,7 @@ func (s3a *S3ApiServer) HeadBucketHandler(w http.ResponseWriter, r *http.Request
|
||||
|
||||
func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorCode {
|
||||
entry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
if entry == nil || errors.Is(err, filer_pb.ErrNotFound) {
|
||||
if entry == nil || err == filer_pb.ErrNotFound {
|
||||
return s3err.ErrNoSuchBucket
|
||||
}
|
||||
|
||||
@@ -669,7 +669,7 @@ func (s3a *S3ApiServer) DeleteBucketOwnershipControls(w http.ResponseWriter, r *
|
||||
|
||||
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
@@ -714,22 +714,11 @@ func (s3a *S3ApiServer) GetBucketVersioningHandler(w http.ResponseWriter, r *htt
|
||||
return
|
||||
}
|
||||
|
||||
// AWS S3 behavior: If versioning was never configured, don't return Status field
|
||||
var response *s3.PutBucketVersioningInput
|
||||
if versioningStatus == "" {
|
||||
// No versioning configuration - return empty response (no Status field)
|
||||
response = &s3.PutBucketVersioningInput{
|
||||
VersioningConfiguration: &s3.VersioningConfiguration{},
|
||||
}
|
||||
} else {
|
||||
// Versioning was explicitly configured - return the status
|
||||
response = &s3.PutBucketVersioningInput{
|
||||
VersioningConfiguration: &s3.VersioningConfiguration{
|
||||
Status: aws.String(versioningStatus),
|
||||
},
|
||||
}
|
||||
}
|
||||
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, response)
|
||||
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, &s3.PutBucketVersioningInput{
|
||||
VersioningConfiguration: &s3.VersioningConfiguration{
|
||||
Status: aws.String(versioningStatus),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// PutBucketVersioningHandler Put bucket Versioning
|
||||
|
||||
@@ -2,7 +2,6 @@ package s3api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -86,95 +85,6 @@ func removeDuplicateSlashes(object string) string {
|
||||
return result.String()
|
||||
}
|
||||
|
||||
// checkDirectoryObject checks if the object is a directory object (ends with "/") and if it exists
|
||||
// Returns: (entry, isDirectoryObject, error)
|
||||
// - entry: the directory entry if found and is a directory
|
||||
// - isDirectoryObject: true if the request was for a directory object (ends with "/")
|
||||
// - error: any error encountered while checking
|
||||
func (s3a *S3ApiServer) checkDirectoryObject(bucket, object string) (*filer_pb.Entry, bool, error) {
|
||||
if !strings.HasSuffix(object, "/") {
|
||||
return nil, false, nil // Not a directory object
|
||||
}
|
||||
|
||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||
cleanObject := strings.TrimSuffix(strings.TrimPrefix(object, "/"), "/")
|
||||
|
||||
if cleanObject == "" {
|
||||
return nil, true, nil // Root level directory object, but we don't handle it
|
||||
}
|
||||
|
||||
// Check if directory exists
|
||||
dirEntry, err := s3a.getEntry(bucketDir, cleanObject)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil, true, nil // Directory object requested but doesn't exist
|
||||
}
|
||||
return nil, true, err // Other errors should be propagated
|
||||
}
|
||||
|
||||
if !dirEntry.IsDirectory {
|
||||
return nil, true, nil // Exists but not a directory
|
||||
}
|
||||
|
||||
return dirEntry, true, nil
|
||||
}
|
||||
|
||||
// serveDirectoryContent serves the content of a directory object directly
|
||||
func (s3a *S3ApiServer) serveDirectoryContent(w http.ResponseWriter, r *http.Request, entry *filer_pb.Entry) {
|
||||
// Set content type - use stored MIME type or default
|
||||
contentType := entry.Attributes.Mime
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
|
||||
// Set content length - use FileSize for accuracy, especially for large files
|
||||
contentLength := int64(entry.Attributes.FileSize)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(contentLength, 10))
|
||||
|
||||
// Set last modified
|
||||
w.Header().Set("Last-Modified", time.Unix(entry.Attributes.Mtime, 0).UTC().Format(http.TimeFormat))
|
||||
|
||||
// Set ETag
|
||||
w.Header().Set("ETag", "\""+filer.ETag(entry)+"\"")
|
||||
|
||||
// For HEAD requests, don't write body
|
||||
if r.Method == http.MethodHead {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
// Write content
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if len(entry.Content) > 0 {
|
||||
if _, err := w.Write(entry.Content); err != nil {
|
||||
glog.Errorf("serveDirectoryContent: failed to write response: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleDirectoryObjectRequest is a helper function that handles directory object requests
|
||||
// for both GET and HEAD operations, eliminating code duplication
|
||||
func (s3a *S3ApiServer) handleDirectoryObjectRequest(w http.ResponseWriter, r *http.Request, bucket, object, handlerName string) bool {
|
||||
// Check if this is a directory object and handle it directly
|
||||
if dirEntry, isDirectoryObject, err := s3a.checkDirectoryObject(bucket, object); err != nil {
|
||||
glog.Errorf("%s: error checking directory object %s/%s: %v", handlerName, bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return true // Request was handled (with error)
|
||||
} else if dirEntry != nil {
|
||||
glog.V(2).Infof("%s: directory object %s/%s found, serving content", handlerName, bucket, object)
|
||||
s3a.serveDirectoryContent(w, r, dirEntry)
|
||||
return true // Request was handled successfully
|
||||
} else if isDirectoryObject {
|
||||
// Directory object but doesn't exist
|
||||
glog.V(2).Infof("%s: directory object %s/%s not found", handlerName, bucket, object)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return true // Request was handled (with not found)
|
||||
}
|
||||
|
||||
return false // Not a directory object, continue with normal processing
|
||||
}
|
||||
|
||||
func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bucketPrefix string, fetchOwner bool, isDirectory bool, encodingTypeUrl bool) (listEntry ListEntry) {
|
||||
storageClass := "STANDARD"
|
||||
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
|
||||
@@ -198,31 +108,9 @@ func newListEntry(entry *filer_pb.Entry, key string, dir string, name string, bu
|
||||
StorageClass: StorageClass(storageClass),
|
||||
}
|
||||
if fetchOwner {
|
||||
// Extract owner from S3 metadata (Extended attributes) instead of file system attributes
|
||||
var ownerID, displayName string
|
||||
if entry.Extended != nil {
|
||||
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
|
||||
ownerID = string(ownerBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to anonymous if no S3 owner found
|
||||
if ownerID == "" {
|
||||
ownerID = s3_constants.AccountAnonymousId
|
||||
displayName = "anonymous"
|
||||
} else {
|
||||
// Use the ownerID as displayName if no better option is available
|
||||
displayName = ownerID
|
||||
|
||||
// Additional fallback to file system username if available and different from ownerID
|
||||
if entry.Attributes.UserName != "" && entry.Attributes.UserName != ownerID {
|
||||
displayName = entry.Attributes.UserName
|
||||
}
|
||||
}
|
||||
|
||||
listEntry.Owner = CanonicalUser{
|
||||
ID: ownerID,
|
||||
DisplayName: displayName,
|
||||
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
|
||||
DisplayName: entry.Attributes.UserName,
|
||||
}
|
||||
}
|
||||
return listEntry
|
||||
@@ -240,16 +128,16 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
bucket, object := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("GetObjectHandler %s %s", bucket, object)
|
||||
|
||||
// Handle directory objects with shared logic
|
||||
if s3a.handleDirectoryObjectRequest(w, r, bucket, object, "GetObjectHandler") {
|
||||
return // Directory object request was handled
|
||||
if strings.HasSuffix(r.URL.Path, "/") {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
|
||||
return
|
||||
}
|
||||
|
||||
// Check for specific version ID in query parameters
|
||||
versionId := r.URL.Query().Get("versionId")
|
||||
|
||||
// Check if versioning is configured for the bucket (Enabled or Suspended)
|
||||
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
|
||||
// Check if versioning is enabled for the bucket
|
||||
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
@@ -260,18 +148,16 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
glog.V(1).Infof("GetObject: bucket %s, object %s, versioningConfigured=%v, versionId=%s", bucket, object, versioningConfigured, versionId)
|
||||
|
||||
var destUrl string
|
||||
|
||||
if versioningConfigured {
|
||||
if versioningEnabled {
|
||||
// Handle versioned GET - all versions are stored in .versions directory
|
||||
var targetVersionId string
|
||||
var entry *filer_pb.Entry
|
||||
|
||||
if versionId != "" {
|
||||
// Request for specific version
|
||||
glog.V(2).Infof("GetObject: requesting specific version %s for %s%s", versionId, bucket, object)
|
||||
glog.V(2).Infof("GetObject: requesting specific version %s for %s/%s", versionId, bucket, object)
|
||||
entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get specific version %s: %v", versionId, err)
|
||||
@@ -281,10 +167,10 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
targetVersionId = versionId
|
||||
} else {
|
||||
// Request for latest version
|
||||
glog.V(1).Infof("GetObject: requesting latest version for %s%s", bucket, object)
|
||||
glog.V(2).Infof("GetObject: requesting latest version for %s/%s", bucket, object)
|
||||
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||
if err != nil {
|
||||
glog.Errorf("GetObject: Failed to get latest version for %s%s: %v", bucket, object, err)
|
||||
glog.Errorf("Failed to get latest version: %v", err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
@@ -293,10 +179,6 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
targetVersionId = string(versionIdBytes)
|
||||
}
|
||||
}
|
||||
// If no version ID found in entry, this is a pre-versioning object
|
||||
if targetVersionId == "" {
|
||||
targetVersionId = "null"
|
||||
}
|
||||
}
|
||||
|
||||
// Check if this is a delete marker
|
||||
@@ -307,17 +189,10 @@ func (s3a *S3ApiServer) GetObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine the actual file path based on whether this is a versioned or pre-versioning object
|
||||
if targetVersionId == "null" {
|
||||
// Pre-versioning object - stored as regular file
|
||||
destUrl = s3a.toFilerUrl(bucket, object)
|
||||
glog.V(2).Infof("GetObject: pre-versioning object URL: %s", destUrl)
|
||||
} else {
|
||||
// Versioned object - stored in .versions directory
|
||||
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
|
||||
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
|
||||
glog.V(2).Infof("GetObject: version %s URL: %s", targetVersionId, destUrl)
|
||||
}
|
||||
// All versions are stored in .versions directory
|
||||
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
|
||||
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
|
||||
glog.V(2).Infof("GetObject: version %s URL: %s", targetVersionId, destUrl)
|
||||
|
||||
// Set version ID in response header
|
||||
w.Header().Set("x-amz-version-id", targetVersionId)
|
||||
@@ -337,16 +212,11 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
bucket, object := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("HeadObjectHandler %s %s", bucket, object)
|
||||
|
||||
// Handle directory objects with shared logic
|
||||
if s3a.handleDirectoryObjectRequest(w, r, bucket, object, "HeadObjectHandler") {
|
||||
return // Directory object request was handled
|
||||
}
|
||||
|
||||
// Check for specific version ID in query parameters
|
||||
versionId := r.URL.Query().Get("versionId")
|
||||
|
||||
// Check if versioning is configured for the bucket (Enabled or Suspended)
|
||||
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
|
||||
// Check if versioning is enabled for the bucket
|
||||
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
@@ -359,14 +229,14 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
|
||||
var destUrl string
|
||||
|
||||
if versioningConfigured {
|
||||
if versioningEnabled {
|
||||
// Handle versioned HEAD - all versions are stored in .versions directory
|
||||
var targetVersionId string
|
||||
var entry *filer_pb.Entry
|
||||
|
||||
if versionId != "" {
|
||||
// Request for specific version
|
||||
glog.V(2).Infof("HeadObject: requesting specific version %s for %s%s", versionId, bucket, object)
|
||||
glog.V(2).Infof("HeadObject: requesting specific version %s for %s/%s", versionId, bucket, object)
|
||||
entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get specific version %s: %v", versionId, err)
|
||||
@@ -376,7 +246,7 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
targetVersionId = versionId
|
||||
} else {
|
||||
// Request for latest version
|
||||
glog.V(2).Infof("HeadObject: requesting latest version for %s%s", bucket, object)
|
||||
glog.V(2).Infof("HeadObject: requesting latest version for %s/%s", bucket, object)
|
||||
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to get latest version: %v", err)
|
||||
@@ -388,10 +258,6 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
targetVersionId = string(versionIdBytes)
|
||||
}
|
||||
}
|
||||
// If no version ID found in entry, this is a pre-versioning object
|
||||
if targetVersionId == "" {
|
||||
targetVersionId = "null"
|
||||
}
|
||||
}
|
||||
|
||||
// Check if this is a delete marker
|
||||
@@ -402,17 +268,10 @@ func (s3a *S3ApiServer) HeadObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
}
|
||||
|
||||
// Determine the actual file path based on whether this is a versioned or pre-versioning object
|
||||
if targetVersionId == "null" {
|
||||
// Pre-versioning object - stored as regular file
|
||||
destUrl = s3a.toFilerUrl(bucket, object)
|
||||
glog.V(2).Infof("HeadObject: pre-versioning object URL: %s", destUrl)
|
||||
} else {
|
||||
// Versioned object - stored in .versions directory
|
||||
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
|
||||
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
|
||||
glog.V(2).Infof("HeadObject: version %s URL: %s", targetVersionId, destUrl)
|
||||
}
|
||||
// All versions are stored in .versions directory
|
||||
versionObjectPath := object + ".versions/" + s3a.getVersionFileName(targetVersionId)
|
||||
destUrl = s3a.toFilerUrl(bucket, versionObjectPath)
|
||||
glog.V(2).Infof("HeadObject: version %s URL: %s", targetVersionId, destUrl)
|
||||
|
||||
// Set version ID in response header
|
||||
w.Header().Set("x-amz-version-id", targetVersionId)
|
||||
|
||||
@@ -1,356 +0,0 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
)
|
||||
|
||||
// GetObjectAclHandler Get object ACL
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html
|
||||
func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// collect parameters
|
||||
bucket, object := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("GetObjectAclHandler %s %s", bucket, object)
|
||||
|
||||
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Check for specific version ID in query parameters
|
||||
versionId := r.URL.Query().Get("versionId")
|
||||
|
||||
// Check if versioning is configured for the bucket (Enabled or Suspended)
|
||||
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
glog.Errorf("GetObjectAclHandler: Error checking versioning status for bucket %s: %v", bucket, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
var entry *filer_pb.Entry
|
||||
|
||||
if versioningConfigured {
|
||||
// Handle versioned object ACL retrieval - use same logic as GetObjectHandler
|
||||
if versionId != "" {
|
||||
// Request for specific version
|
||||
glog.V(2).Infof("GetObjectAclHandler: requesting ACL for specific version %s of %s%s", versionId, bucket, object)
|
||||
entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
|
||||
} else {
|
||||
// Request for latest version
|
||||
glog.V(2).Infof("GetObjectAclHandler: requesting ACL for latest version of %s%s", bucket, object)
|
||||
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("GetObjectAclHandler: Failed to get object version %s for %s%s: %v", versionId, bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if this is a delete marker
|
||||
if entry.Extended != nil {
|
||||
if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Handle regular (non-versioned) object ACL retrieval
|
||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||
entry, err = s3a.getEntry(bucketDir, object)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
glog.Errorf("GetObjectAclHandler: error checking object %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if entry == nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
|
||||
// Get object owner from metadata, fallback to request account
|
||||
var objectOwner string
|
||||
var objectOwnerDisplayName string
|
||||
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
|
||||
|
||||
if entry.Extended != nil {
|
||||
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
|
||||
objectOwner = string(ownerBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to current account if no owner stored
|
||||
if objectOwner == "" {
|
||||
objectOwner = amzAccountId
|
||||
}
|
||||
|
||||
objectOwnerDisplayName = s3a.iam.GetAccountNameById(objectOwner)
|
||||
|
||||
// Build ACL response
|
||||
response := AccessControlPolicy{
|
||||
Owner: CanonicalUser{
|
||||
ID: objectOwner,
|
||||
DisplayName: objectOwnerDisplayName,
|
||||
},
|
||||
}
|
||||
|
||||
// Get grants from stored ACL metadata
|
||||
grants := GetAcpGrants(entry.Extended)
|
||||
if len(grants) > 0 {
|
||||
// Convert AWS SDK grants to local Grant format
|
||||
for _, grant := range grants {
|
||||
localGrant := Grant{
|
||||
Permission: Permission(*grant.Permission),
|
||||
}
|
||||
|
||||
if grant.Grantee != nil {
|
||||
localGrant.Grantee = Grantee{
|
||||
Type: *grant.Grantee.Type,
|
||||
XMLXSI: "CanonicalUser",
|
||||
XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
|
||||
}
|
||||
|
||||
if grant.Grantee.ID != nil {
|
||||
localGrant.Grantee.ID = *grant.Grantee.ID
|
||||
localGrant.Grantee.DisplayName = s3a.iam.GetAccountNameById(*grant.Grantee.ID)
|
||||
}
|
||||
|
||||
if grant.Grantee.URI != nil {
|
||||
localGrant.Grantee.URI = *grant.Grantee.URI
|
||||
}
|
||||
}
|
||||
|
||||
response.AccessControlList.Grant = append(response.AccessControlList.Grant, localGrant)
|
||||
}
|
||||
} else {
|
||||
// Fallback to default full control for object owner
|
||||
response.AccessControlList.Grant = append(response.AccessControlList.Grant, Grant{
|
||||
Grantee: Grantee{
|
||||
ID: objectOwner,
|
||||
DisplayName: objectOwnerDisplayName,
|
||||
Type: "CanonicalUser",
|
||||
XMLXSI: "CanonicalUser",
|
||||
XMLNS: "http://www.w3.org/2001/XMLSchema-instance"},
|
||||
Permission: Permission(s3_constants.PermissionFullControl),
|
||||
})
|
||||
}
|
||||
|
||||
writeSuccessResponseXML(w, r, response)
|
||||
}
|
||||
|
||||
// PutObjectAclHandler Put object ACL
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html
|
||||
func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// collect parameters
|
||||
bucket, object := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("PutObjectAclHandler %s %s", bucket, object)
|
||||
|
||||
if err := s3a.checkBucket(r, bucket); err != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Check for specific version ID in query parameters
|
||||
versionId := r.URL.Query().Get("versionId")
|
||||
|
||||
// Check if versioning is configured for the bucket (Enabled or Suspended)
|
||||
versioningConfigured, err := s3a.isVersioningConfigured(bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
glog.Errorf("PutObjectAclHandler: Error checking versioning status for bucket %s: %v", bucket, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
var entry *filer_pb.Entry
|
||||
|
||||
if versioningConfigured {
|
||||
// Handle versioned object ACL modification - use same logic as GetObjectHandler
|
||||
if versionId != "" {
|
||||
// Request for specific version
|
||||
glog.V(2).Infof("PutObjectAclHandler: modifying ACL for specific version %s of %s%s", versionId, bucket, object)
|
||||
entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId)
|
||||
} else {
|
||||
// Request for latest version
|
||||
glog.V(2).Infof("PutObjectAclHandler: modifying ACL for latest version of %s%s", bucket, object)
|
||||
entry, err = s3a.getLatestObjectVersion(bucket, object)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("PutObjectAclHandler: Failed to get object version %s for %s%s: %v", versionId, bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if this is a delete marker
|
||||
if entry.Extended != nil {
|
||||
if deleteMarker, exists := entry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Handle regular (non-versioned) object ACL modification
|
||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||
entry, err = s3a.getEntry(bucketDir, object)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
glog.Errorf("PutObjectAclHandler: error checking object %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if entry == nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchKey)
|
||||
return
|
||||
}
|
||||
|
||||
// Get current object owner from metadata
|
||||
var objectOwner string
|
||||
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
|
||||
|
||||
if entry.Extended != nil {
|
||||
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
|
||||
objectOwner = string(ownerBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to current account if no owner stored
|
||||
if objectOwner == "" {
|
||||
objectOwner = amzAccountId
|
||||
}
|
||||
|
||||
// **PERMISSION CHECKS**
|
||||
|
||||
// 1. Check if user is admin (admins can modify any ACL)
|
||||
if !s3a.isUserAdmin(r) {
|
||||
// 2. Check object ownership - only object owner can modify ACL (unless admin)
|
||||
if objectOwner != amzAccountId {
|
||||
glog.V(3).Infof("PutObjectAclHandler: Access denied - user %s is not owner of object %s/%s (owner: %s)",
|
||||
amzAccountId, bucket, object, objectOwner)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Check object-level WRITE_ACP permission
|
||||
// Create the specific action for this object
|
||||
writeAcpAction := Action(fmt.Sprintf("WriteAcp:%s/%s", bucket, object))
|
||||
identity, errCode := s3a.iam.authRequest(r, writeAcpAction)
|
||||
if errCode != s3err.ErrNone {
|
||||
glog.V(3).Infof("PutObjectAclHandler: Auth failed for WriteAcp action on %s/%s: %v", bucket, object, errCode)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
|
||||
// 4. Verify the authenticated identity can perform WriteAcp on this specific object
|
||||
if identity == nil || !identity.canDo(writeAcpAction, bucket, object) {
|
||||
glog.V(3).Infof("PutObjectAclHandler: Identity %v cannot perform WriteAcp on %s/%s", identity, bucket, object)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
glog.V(3).Infof("PutObjectAclHandler: Admin user %s granted ACL modification permission for %s/%s", amzAccountId, bucket, object)
|
||||
}
|
||||
|
||||
// Get bucket config for ownership settings
|
||||
bucketConfig, errCode := s3a.getBucketConfig(bucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
bucketOwnership := bucketConfig.Ownership
|
||||
bucketOwnerId := bucketConfig.Owner
|
||||
|
||||
// Extract ACL from request (either canned ACL or XML body)
|
||||
// This function also validates that the owner in the request matches the object owner
|
||||
grants, errCode := ExtractAcl(r, s3a.iam, bucketOwnership, bucketOwnerId, objectOwner, amzAccountId)
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Store ACL in object metadata
|
||||
if errCode := AssembleEntryWithAcp(entry, objectOwner, grants); errCode != s3err.ErrNone {
|
||||
glog.Errorf("PutObjectAclHandler: failed to assemble entry with ACP: %v", errCode)
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate the correct directory for ACL update
|
||||
var updateDirectory string
|
||||
|
||||
if versioningConfigured {
|
||||
if versionId != "" && versionId != "null" {
|
||||
// Versioned object - update the specific version file in .versions directory
|
||||
updateDirectory = s3a.option.BucketsPath + "/" + bucket + "/" + object + ".versions"
|
||||
} else {
|
||||
// Latest version in versioned bucket - could be null version or versioned object
|
||||
// Extract version ID from the entry to determine where it's stored
|
||||
var actualVersionId string
|
||||
if entry.Extended != nil {
|
||||
if versionIdBytes, exists := entry.Extended[s3_constants.ExtVersionIdKey]; exists {
|
||||
actualVersionId = string(versionIdBytes)
|
||||
}
|
||||
}
|
||||
|
||||
if actualVersionId == "null" || actualVersionId == "" {
|
||||
// Null version (pre-versioning object) - stored as regular file
|
||||
updateDirectory = s3a.option.BucketsPath + "/" + bucket
|
||||
} else {
|
||||
// Versioned object - stored in .versions directory
|
||||
updateDirectory = s3a.option.BucketsPath + "/" + bucket + "/" + object + ".versions"
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Non-versioned object - stored as regular file
|
||||
updateDirectory = s3a.option.BucketsPath + "/" + bucket
|
||||
}
|
||||
|
||||
// Update the object with new ACL metadata
|
||||
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
request := &filer_pb.UpdateEntryRequest{
|
||||
Directory: updateDirectory,
|
||||
Entry: entry,
|
||||
}
|
||||
|
||||
if _, err := client.UpdateEntry(context.Background(), request); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("PutObjectAclHandler: failed to update entry: %v", err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
glog.V(3).Infof("PutObjectAclHandler: Successfully updated ACL for %s/%s by user %s", bucket, object, amzAccountId)
|
||||
writeSuccessResponseEmpty(w, r)
|
||||
}
|
||||
@@ -38,9 +38,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
|
||||
}
|
||||
|
||||
srcBucket, srcObject, srcVersionId := pathToBucketObjectAndVersion(cpSrcPath)
|
||||
srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
|
||||
|
||||
glog.V(3).Infof("CopyObjectHandler %s %s (version: %s) => %s %s", srcBucket, srcObject, srcVersionId, dstBucket, dstObject)
|
||||
glog.V(3).Infof("CopyObjectHandler %s %s => %s %s", srcBucket, srcObject, dstBucket, dstObject)
|
||||
|
||||
replaceMeta, replaceTagging := replaceDirective(r.Header)
|
||||
|
||||
@@ -76,41 +76,9 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
||||
return
|
||||
}
|
||||
|
||||
// Get detailed versioning state for source bucket
|
||||
srcVersioningState, err := s3a.getVersioningState(srcBucket)
|
||||
if err != nil {
|
||||
glog.Errorf("Error checking versioning state for source bucket %s: %v", srcBucket, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
||||
return
|
||||
}
|
||||
|
||||
// Get the source entry with version awareness based on versioning state
|
||||
var entry *filer_pb.Entry
|
||||
if srcVersionId != "" {
|
||||
// Specific version requested - always use version-aware retrieval
|
||||
entry, err = s3a.getSpecificObjectVersion(srcBucket, srcObject, srcVersionId)
|
||||
} else if srcVersioningState == s3_constants.VersioningEnabled {
|
||||
// Versioning enabled - get latest version from .versions directory
|
||||
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
|
||||
} else if srcVersioningState == s3_constants.VersioningSuspended {
|
||||
// Versioning suspended - current object is stored as regular file ("null" version)
|
||||
// Try regular file first, fall back to latest version if needed
|
||||
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
|
||||
dir, name := srcPath.DirAndName()
|
||||
entry, err = s3a.getEntry(dir, name)
|
||||
if err != nil {
|
||||
// If regular file doesn't exist, try latest version as fallback
|
||||
glog.V(2).Infof("CopyObject: regular file not found for suspended versioning, trying latest version")
|
||||
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
|
||||
}
|
||||
} else {
|
||||
// No versioning configured - use regular retrieval
|
||||
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
|
||||
dir, name := srcPath.DirAndName()
|
||||
entry, err = s3a.getEntry(dir, name)
|
||||
}
|
||||
|
||||
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
|
||||
dir, name := srcPath.DirAndName()
|
||||
entry, err := s3a.getEntry(dir, name)
|
||||
if err != nil || entry.IsDirectory {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
||||
return
|
||||
@@ -170,108 +138,43 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
dstEntry.Chunks = dstChunks
|
||||
}
|
||||
|
||||
// Check if destination bucket has versioning configured
|
||||
dstVersioningConfigured, err := s3a.isVersioningConfigured(dstBucket)
|
||||
if err != nil {
|
||||
glog.Errorf("Error checking versioning status for destination bucket %s: %v", dstBucket, err)
|
||||
// Save the new entry
|
||||
dstPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
|
||||
dstDir, dstName := dstPath.DirAndName()
|
||||
|
||||
// Check if destination exists and remove it first (S3 copy overwrites)
|
||||
if exists, _ := s3a.exists(dstDir, dstName, false); exists {
|
||||
if err := s3a.rm(dstDir, dstName, false, false); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Create the new file
|
||||
if err := s3a.mkFile(dstDir, dstName, dstEntry.Chunks, func(entry *filer_pb.Entry) {
|
||||
entry.Attributes = dstEntry.Attributes
|
||||
entry.Extended = dstEntry.Extended
|
||||
}); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
var dstVersionId string
|
||||
var etag string
|
||||
|
||||
if dstVersioningConfigured {
|
||||
// For versioned destination, create a new version
|
||||
dstVersionId = generateVersionId()
|
||||
glog.V(2).Infof("CopyObjectHandler: creating version %s for destination %s/%s", dstVersionId, dstBucket, dstObject)
|
||||
|
||||
// Add version metadata to the entry
|
||||
if dstEntry.Extended == nil {
|
||||
dstEntry.Extended = make(map[string][]byte)
|
||||
}
|
||||
dstEntry.Extended[s3_constants.ExtVersionIdKey] = []byte(dstVersionId)
|
||||
|
||||
// Calculate ETag for versioning
|
||||
filerEntry := &filer.Entry{
|
||||
FullPath: util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject)),
|
||||
Attr: filer.Attr{
|
||||
FileSize: dstEntry.Attributes.FileSize,
|
||||
Mtime: time.Unix(dstEntry.Attributes.Mtime, 0),
|
||||
Crtime: time.Unix(dstEntry.Attributes.Crtime, 0),
|
||||
Mime: dstEntry.Attributes.Mime,
|
||||
},
|
||||
Chunks: dstEntry.Chunks,
|
||||
}
|
||||
etag = filer.ETagEntry(filerEntry)
|
||||
if !strings.HasPrefix(etag, "\"") {
|
||||
etag = "\"" + etag + "\""
|
||||
}
|
||||
dstEntry.Extended[s3_constants.ExtETagKey] = []byte(etag)
|
||||
|
||||
// Create version file
|
||||
versionFileName := s3a.getVersionFileName(dstVersionId)
|
||||
versionObjectPath := dstObject + ".versions/" + versionFileName
|
||||
bucketDir := s3a.option.BucketsPath + "/" + dstBucket
|
||||
|
||||
if err := s3a.mkFile(bucketDir, versionObjectPath, dstEntry.Chunks, func(entry *filer_pb.Entry) {
|
||||
entry.Attributes = dstEntry.Attributes
|
||||
entry.Extended = dstEntry.Extended
|
||||
}); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
// Update the .versions directory metadata
|
||||
err = s3a.updateLatestVersionInDirectory(dstBucket, dstObject, dstVersionId, versionFileName)
|
||||
if err != nil {
|
||||
glog.Errorf("CopyObjectHandler: failed to update latest version in directory: %v", err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
// Set version ID in response header
|
||||
w.Header().Set("x-amz-version-id", dstVersionId)
|
||||
} else {
|
||||
// For non-versioned destination, use regular copy
|
||||
dstPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, dstBucket, dstObject))
|
||||
dstDir, dstName := dstPath.DirAndName()
|
||||
|
||||
// Check if destination exists and remove it first (S3 copy overwrites)
|
||||
if exists, _ := s3a.exists(dstDir, dstName, false); exists {
|
||||
if err := s3a.rm(dstDir, dstName, false, false); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Create the new file
|
||||
if err := s3a.mkFile(dstDir, dstName, dstEntry.Chunks, func(entry *filer_pb.Entry) {
|
||||
entry.Attributes = dstEntry.Attributes
|
||||
entry.Extended = dstEntry.Extended
|
||||
}); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate ETag
|
||||
filerEntry := &filer.Entry{
|
||||
FullPath: dstPath,
|
||||
Attr: filer.Attr{
|
||||
FileSize: dstEntry.Attributes.FileSize,
|
||||
Mtime: time.Unix(dstEntry.Attributes.Mtime, 0),
|
||||
Crtime: time.Unix(dstEntry.Attributes.Crtime, 0),
|
||||
Mime: dstEntry.Attributes.Mime,
|
||||
},
|
||||
Chunks: dstEntry.Chunks,
|
||||
}
|
||||
etag = filer.ETagEntry(filerEntry)
|
||||
// Convert filer_pb.Entry to filer.Entry for ETag calculation
|
||||
filerEntry := &filer.Entry{
|
||||
FullPath: dstPath,
|
||||
Attr: filer.Attr{
|
||||
FileSize: dstEntry.Attributes.FileSize,
|
||||
Mtime: time.Unix(dstEntry.Attributes.Mtime, 0),
|
||||
Crtime: time.Unix(dstEntry.Attributes.Crtime, 0),
|
||||
Mime: dstEntry.Attributes.Mime,
|
||||
},
|
||||
Chunks: dstEntry.Chunks,
|
||||
}
|
||||
|
||||
setEtag(w, etag)
|
||||
setEtag(w, filer.ETagEntry(filerEntry))
|
||||
|
||||
response := CopyObjectResult{
|
||||
ETag: etag,
|
||||
ETag: filer.ETagEntry(filerEntry),
|
||||
LastModified: time.Now().UTC(),
|
||||
}
|
||||
|
||||
@@ -288,18 +191,6 @@ func pathToBucketAndObject(path string) (bucket, object string) {
|
||||
return parts[0], "/"
|
||||
}
|
||||
|
||||
func pathToBucketObjectAndVersion(path string) (bucket, object, versionId string) {
|
||||
// Parse versionId from query string if present
|
||||
// Format: /bucket/object?versionId=version-id
|
||||
if idx := strings.Index(path, "?versionId="); idx != -1 {
|
||||
versionId = path[idx+len("?versionId="):] // dynamically calculate length
|
||||
path = path[:idx]
|
||||
}
|
||||
|
||||
bucket, object = pathToBucketAndObject(path)
|
||||
return bucket, object, versionId
|
||||
}
|
||||
|
||||
type CopyPartResult struct {
|
||||
LastModified time.Time `xml:"LastModified"`
|
||||
ETag string `xml:"ETag"`
|
||||
@@ -317,7 +208,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
|
||||
cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
|
||||
}
|
||||
|
||||
srcBucket, srcObject, srcVersionId := pathToBucketObjectAndVersion(cpSrcPath)
|
||||
srcBucket, srcObject := pathToBucketAndObject(cpSrcPath)
|
||||
// If source object is empty or bucket is empty, reply back invalid copy source.
|
||||
if srcObject == "" || srcBucket == "" {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
||||
@@ -348,40 +239,10 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
// Get detailed versioning state for source bucket
|
||||
srcVersioningState, err := s3a.getVersioningState(srcBucket)
|
||||
if err != nil {
|
||||
glog.Errorf("Error checking versioning state for source bucket %s: %v", srcBucket, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
||||
return
|
||||
}
|
||||
|
||||
// Get the source entry with version awareness based on versioning state
|
||||
var entry *filer_pb.Entry
|
||||
if srcVersionId != "" {
|
||||
// Specific version requested - always use version-aware retrieval
|
||||
entry, err = s3a.getSpecificObjectVersion(srcBucket, srcObject, srcVersionId)
|
||||
} else if srcVersioningState == s3_constants.VersioningEnabled {
|
||||
// Versioning enabled - get latest version from .versions directory
|
||||
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
|
||||
} else if srcVersioningState == s3_constants.VersioningSuspended {
|
||||
// Versioning suspended - current object is stored as regular file ("null" version)
|
||||
// Try regular file first, fall back to latest version if needed
|
||||
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
|
||||
dir, name := srcPath.DirAndName()
|
||||
entry, err = s3a.getEntry(dir, name)
|
||||
if err != nil {
|
||||
// If regular file doesn't exist, try latest version as fallback
|
||||
glog.V(2).Infof("CopyObjectPart: regular file not found for suspended versioning, trying latest version")
|
||||
entry, err = s3a.getLatestObjectVersion(srcBucket, srcObject)
|
||||
}
|
||||
} else {
|
||||
// No versioning configured - use regular retrieval
|
||||
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
|
||||
dir, name := srcPath.DirAndName()
|
||||
entry, err = s3a.getEntry(dir, name)
|
||||
}
|
||||
|
||||
// Get source entry
|
||||
srcPath := util.FullPath(fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, srcBucket, srcObject))
|
||||
dir, name := srcPath.DirAndName()
|
||||
entry, err := s3a.getEntry(dir, name)
|
||||
if err != nil || entry.IsDirectory {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopySource)
|
||||
return
|
||||
|
||||
@@ -32,8 +32,8 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
||||
// Check for specific version ID in query parameters
|
||||
versionId := r.URL.Query().Get("versionId")
|
||||
|
||||
// Get detailed versioning state for proper handling of suspended vs enabled versioning
|
||||
versioningState, err := s3a.getVersioningState(bucket)
|
||||
// Check if versioning is enabled for the bucket
|
||||
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
@@ -44,19 +44,14 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
versioningEnabled := (versioningState == s3_constants.VersioningEnabled)
|
||||
versioningSuspended := (versioningState == s3_constants.VersioningSuspended)
|
||||
versioningConfigured := (versioningState != "")
|
||||
|
||||
var auditLog *s3err.AccessLog
|
||||
if s3err.Logger != nil {
|
||||
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
|
||||
}
|
||||
|
||||
if versioningConfigured {
|
||||
// Handle versioned delete based on specific versioning state
|
||||
if versioningEnabled {
|
||||
// Handle versioned delete
|
||||
if versionId != "" {
|
||||
// Delete specific version (same for both enabled and suspended)
|
||||
// Check object lock permissions before deleting specific version
|
||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
|
||||
if err := s3a.enforceObjectLockProtections(r, bucket, object, versionId, governanceBypassAllowed); err != nil {
|
||||
@@ -76,44 +71,19 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
||||
// Set version ID in response header
|
||||
w.Header().Set("x-amz-version-id", versionId)
|
||||
} else {
|
||||
// Delete without version ID - behavior depends on versioning state
|
||||
if versioningEnabled {
|
||||
// Enabled versioning: Create delete marker (logical delete)
|
||||
// AWS S3 behavior: Delete marker creation is NOT blocked by object retention
|
||||
// because it's a logical delete that doesn't actually remove the retained version
|
||||
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to create delete marker: %v", err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
// Set delete marker version ID in response header
|
||||
w.Header().Set("x-amz-version-id", deleteMarkerVersionId)
|
||||
w.Header().Set("x-amz-delete-marker", "true")
|
||||
} else if versioningSuspended {
|
||||
// Suspended versioning: Actually delete the "null" version object
|
||||
glog.V(2).Infof("DeleteObjectHandler: deleting null version for suspended versioning %s/%s", bucket, object)
|
||||
|
||||
// Check object lock permissions before deleting "null" version
|
||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
|
||||
if err := s3a.enforceObjectLockProtections(r, bucket, object, "null", governanceBypassAllowed); err != nil {
|
||||
glog.V(2).Infof("DeleteObjectHandler: object lock check failed for %s/%s: %v", bucket, object, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
|
||||
return
|
||||
}
|
||||
|
||||
// Delete the "null" version (the regular file)
|
||||
err := s3a.deleteSpecificObjectVersion(bucket, object, "null")
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to delete null version: %v", err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
// Note: According to AWS S3 spec, suspended versioning should NOT return version ID headers
|
||||
// The object is deleted but no version information is returned
|
||||
// Create delete marker (logical delete)
|
||||
// AWS S3 behavior: Delete marker creation is NOT blocked by object retention
|
||||
// because it's a logical delete that doesn't actually remove the retained version
|
||||
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object)
|
||||
if err != nil {
|
||||
glog.Errorf("Failed to create delete marker: %v", err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
// Set delete marker version ID in response header
|
||||
w.Header().Set("x-amz-version-id", deleteMarkerVersionId)
|
||||
w.Header().Set("x-amz-delete-marker", "true")
|
||||
}
|
||||
} else {
|
||||
// Handle regular delete (non-versioned)
|
||||
@@ -167,10 +137,8 @@ func (s3a *S3ApiServer) DeleteObjectHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
// ObjectIdentifier represents an object to be deleted with its key name and optional version ID.
|
||||
type ObjectIdentifier struct {
|
||||
Key string `xml:"Key"`
|
||||
VersionId string `xml:"VersionId,omitempty"`
|
||||
DeleteMarker bool `xml:"DeleteMarker,omitempty"`
|
||||
DeleteMarkerVersionId string `xml:"DeleteMarkerVersionId,omitempty"`
|
||||
Key string `xml:"Key"`
|
||||
VersionId string `xml:"VersionId,omitempty"`
|
||||
}
|
||||
|
||||
// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
|
||||
@@ -233,8 +201,8 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
||||
auditLog = s3err.GetAccessLog(r, http.StatusNoContent, s3err.ErrNone)
|
||||
}
|
||||
|
||||
// Get detailed versioning state for proper handling of suspended vs enabled versioning
|
||||
versioningState, err := s3a.getVersioningState(bucket)
|
||||
// Check if versioning is enabled for the bucket (needed for object lock checks)
|
||||
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
@@ -245,10 +213,6 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
versioningEnabled := (versioningState == s3_constants.VersioningEnabled)
|
||||
versioningSuspended := (versioningState == s3_constants.VersioningSuspended)
|
||||
versioningConfigured := (versioningState != "")
|
||||
|
||||
s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
// delete file entries
|
||||
@@ -258,7 +222,7 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
|
||||
// Check object lock permissions before deletion (only for versioned buckets)
|
||||
if versioningConfigured {
|
||||
if versioningEnabled {
|
||||
// Validate governance bypass for this specific object
|
||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object.Key)
|
||||
if err := s3a.enforceObjectLockProtections(r, bucket, object.Key, object.VersionId, governanceBypassAllowed); err != nil {
|
||||
@@ -272,110 +236,31 @@ func (s3a *S3ApiServer) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *h
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
var deleteVersionId string
|
||||
var isDeleteMarker bool
|
||||
|
||||
if versioningConfigured {
|
||||
// Handle versioned delete based on specific versioning state
|
||||
if object.VersionId != "" {
|
||||
// Delete specific version (same for both enabled and suspended)
|
||||
err := s3a.deleteSpecificObjectVersion(bucket, object.Key, object.VersionId)
|
||||
if err != nil {
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: "",
|
||||
Message: err.Error(),
|
||||
Key: object.Key,
|
||||
VersionId: object.VersionId,
|
||||
})
|
||||
continue
|
||||
}
|
||||
deleteVersionId = object.VersionId
|
||||
} else {
|
||||
// Delete without version ID - behavior depends on versioning state
|
||||
if versioningEnabled {
|
||||
// Enabled versioning: Create delete marker (logical delete)
|
||||
deleteMarkerVersionId, err := s3a.createDeleteMarker(bucket, object.Key)
|
||||
if err != nil {
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: "",
|
||||
Message: err.Error(),
|
||||
Key: object.Key,
|
||||
VersionId: object.VersionId,
|
||||
})
|
||||
continue
|
||||
}
|
||||
deleteVersionId = deleteMarkerVersionId
|
||||
isDeleteMarker = true
|
||||
} else if versioningSuspended {
|
||||
// Suspended versioning: Actually delete the "null" version object
|
||||
glog.V(2).Infof("DeleteMultipleObjectsHandler: deleting null version for suspended versioning %s/%s", bucket, object.Key)
|
||||
|
||||
err := s3a.deleteSpecificObjectVersion(bucket, object.Key, "null")
|
||||
if err != nil {
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: "",
|
||||
Message: err.Error(),
|
||||
Key: object.Key,
|
||||
VersionId: "null",
|
||||
})
|
||||
continue
|
||||
}
|
||||
deleteVersionId = "null"
|
||||
// Note: For suspended versioning, we don't set isDeleteMarker=true
|
||||
// because we actually deleted the object, not created a delete marker
|
||||
}
|
||||
}
|
||||
|
||||
// Add to successful deletions with version info
|
||||
deletedObject := ObjectIdentifier{
|
||||
Key: object.Key,
|
||||
VersionId: deleteVersionId,
|
||||
DeleteMarker: isDeleteMarker,
|
||||
}
|
||||
|
||||
// For delete markers, also set DeleteMarkerVersionId field
|
||||
if isDeleteMarker {
|
||||
deletedObject.DeleteMarkerVersionId = deleteVersionId
|
||||
// Don't set VersionId for delete markers, use DeleteMarkerVersionId instead
|
||||
deletedObject.VersionId = ""
|
||||
}
|
||||
if !deleteObjects.Quiet {
|
||||
deletedObjects = append(deletedObjects, deletedObject)
|
||||
}
|
||||
if isDeleteMarker {
|
||||
// For delete markers, we don't need to track directories for cleanup
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
// Handle non-versioned delete (original logic)
|
||||
lastSeparator := strings.LastIndex(object.Key, "/")
|
||||
parentDirectoryPath, entryName, isDeleteData, isRecursive := "", object.Key, true, false
|
||||
if lastSeparator > 0 && lastSeparator+1 < len(object.Key) {
|
||||
entryName = object.Key[lastSeparator+1:]
|
||||
parentDirectoryPath = "/" + object.Key[:lastSeparator]
|
||||
}
|
||||
parentDirectoryPath = fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, parentDirectoryPath)
|
||||
|
||||
err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
|
||||
if err == nil {
|
||||
directoriesWithDeletion[parentDirectoryPath]++
|
||||
deletedObjects = append(deletedObjects, object)
|
||||
} else if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
|
||||
deletedObjects = append(deletedObjects, object)
|
||||
} else {
|
||||
delete(directoriesWithDeletion, parentDirectoryPath)
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: "",
|
||||
Message: err.Error(),
|
||||
Key: object.Key,
|
||||
VersionId: object.VersionId,
|
||||
})
|
||||
}
|
||||
lastSeparator := strings.LastIndex(object.Key, "/")
|
||||
parentDirectoryPath, entryName, isDeleteData, isRecursive := "", object.Key, true, false
|
||||
if lastSeparator > 0 && lastSeparator+1 < len(object.Key) {
|
||||
entryName = object.Key[lastSeparator+1:]
|
||||
parentDirectoryPath = "/" + object.Key[:lastSeparator]
|
||||
}
|
||||
parentDirectoryPath = fmt.Sprintf("%s/%s%s", s3a.option.BucketsPath, bucket, parentDirectoryPath)
|
||||
|
||||
err := doDeleteEntry(client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
|
||||
if err == nil {
|
||||
directoriesWithDeletion[parentDirectoryPath]++
|
||||
deletedObjects = append(deletedObjects, object)
|
||||
} else if strings.Contains(err.Error(), filer.MsgFailDelNonEmptyFolder) {
|
||||
deletedObjects = append(deletedObjects, object)
|
||||
} else {
|
||||
delete(directoriesWithDeletion, parentDirectoryPath)
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: "",
|
||||
Message: err.Error(),
|
||||
Key: object.Key,
|
||||
VersionId: object.VersionId,
|
||||
})
|
||||
}
|
||||
if auditLog != nil {
|
||||
auditLog.Key = object.Key
|
||||
auditLog.Key = entryName
|
||||
s3err.PostAccessLog(*auditLog)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,17 +4,16 @@ import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type OptionalString struct {
|
||||
@@ -148,32 +147,10 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
||||
prefixEndsOnDelimiter: strings.HasSuffix(originalPrefix, "/") && len(originalMarker) == 0,
|
||||
}
|
||||
|
||||
// Special case: when maxKeys = 0, return empty results immediately
|
||||
if maxKeys == 0 {
|
||||
response = ListBucketResult{
|
||||
Name: bucket,
|
||||
Prefix: originalPrefix,
|
||||
Marker: originalMarker,
|
||||
NextMarker: "",
|
||||
MaxKeys: int(maxKeys),
|
||||
Delimiter: delimiter,
|
||||
IsTruncated: false,
|
||||
Contents: contents,
|
||||
CommonPrefixes: commonPrefixes,
|
||||
}
|
||||
if encodingTypeUrl {
|
||||
response.EncodingType = s3.EncodingTypeUrl
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// check filer
|
||||
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
for {
|
||||
empty := true
|
||||
var lastEntryWasCommonPrefix bool
|
||||
var lastCommonPrefixName string
|
||||
|
||||
nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
|
||||
empty = false
|
||||
dirName, entryName, prefixName := entryUrlEncode(dir, entry.Name, encodingTypeUrl)
|
||||
@@ -181,7 +158,6 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
||||
if entry.IsDirectoryKeyObject() {
|
||||
contents = append(contents, newListEntry(entry, "", dirName, entryName, bucketPrefix, fetchOwner, true, false))
|
||||
cursor.maxKeys--
|
||||
lastEntryWasCommonPrefix = false
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
|
||||
} else if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
|
||||
commonPrefixes = append(commonPrefixes, PrefixEntry{
|
||||
@@ -189,8 +165,6 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
||||
})
|
||||
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
|
||||
cursor.maxKeys--
|
||||
lastEntryWasCommonPrefix = true
|
||||
lastCommonPrefixName = entry.Name
|
||||
}
|
||||
} else {
|
||||
var delimiterFound bool
|
||||
@@ -221,15 +195,12 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
||||
})
|
||||
cursor.maxKeys--
|
||||
delimiterFound = true
|
||||
lastEntryWasCommonPrefix = true
|
||||
lastCommonPrefixName = delimitedPath[0]
|
||||
}
|
||||
}
|
||||
}
|
||||
if !delimiterFound {
|
||||
contents = append(contents, newListEntry(entry, "", dirName, entryName, bucketPrefix, fetchOwner, false, false))
|
||||
cursor.maxKeys--
|
||||
lastEntryWasCommonPrefix = false
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -237,21 +208,10 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
||||
return doErr
|
||||
}
|
||||
|
||||
// Adjust nextMarker for CommonPrefixes to include trailing slash (AWS S3 compliance)
|
||||
if cursor.isTruncated && lastEntryWasCommonPrefix && lastCommonPrefixName != "" {
|
||||
// For CommonPrefixes, NextMarker should include the trailing slash
|
||||
if requestDir != "" {
|
||||
nextMarker = requestDir + "/" + lastCommonPrefixName + "/"
|
||||
} else {
|
||||
nextMarker = lastCommonPrefixName + "/"
|
||||
}
|
||||
} else if cursor.isTruncated {
|
||||
if cursor.isTruncated {
|
||||
if requestDir != "" {
|
||||
nextMarker = requestDir + "/" + nextMarker
|
||||
}
|
||||
}
|
||||
|
||||
if cursor.isTruncated {
|
||||
break
|
||||
} else if empty || strings.HasSuffix(originalPrefix, "/") {
|
||||
nextMarker = ""
|
||||
@@ -396,9 +356,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
||||
return
|
||||
}
|
||||
|
||||
// Track .versions directories found in this directory for later processing
|
||||
var versionsDirs []string
|
||||
|
||||
for {
|
||||
resp, recvErr := stream.Recv()
|
||||
if recvErr != nil {
|
||||
@@ -410,18 +367,8 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
||||
}
|
||||
}
|
||||
if cursor.maxKeys <= 0 {
|
||||
// Check if there are more entries available by trying to peek at the next one
|
||||
_, nextRecvErr := stream.Recv()
|
||||
if nextRecvErr == nil {
|
||||
// There is another entry available, so we're truncated
|
||||
cursor.isTruncated = true
|
||||
} else if nextRecvErr != io.EOF {
|
||||
// Some other error occurred
|
||||
err = fmt.Errorf("peeking next entry: %v", nextRecvErr)
|
||||
return
|
||||
}
|
||||
// If nextRecvErr == io.EOF, there are no more entries, so isTruncated remains false
|
||||
break
|
||||
cursor.isTruncated = true
|
||||
continue
|
||||
}
|
||||
entry := resp.Entry
|
||||
nextMarker = entry.Name
|
||||
@@ -439,14 +386,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
||||
if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip .versions directories in regular list operations but track them for logical object creation
|
||||
if strings.HasSuffix(entry.Name, ".versions") {
|
||||
glog.V(4).Infof("Found .versions directory: %s", entry.Name)
|
||||
versionsDirs = append(versionsDirs, entry.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
if delimiter != "/" || cursor.prefixEndsOnDelimiter {
|
||||
if cursor.prefixEndsOnDelimiter {
|
||||
cursor.prefixEndsOnDelimiter = false
|
||||
@@ -486,49 +425,6 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
|
||||
cursor.prefixEndsOnDelimiter = false
|
||||
}
|
||||
}
|
||||
|
||||
// After processing all regular entries, handle versioned objects
|
||||
// Create logical entries for objects that have .versions directories
|
||||
for _, versionsDir := range versionsDirs {
|
||||
if cursor.maxKeys <= 0 {
|
||||
// We have versioned objects remaining but can't process them due to limit
|
||||
cursor.isTruncated = true
|
||||
break
|
||||
}
|
||||
|
||||
// Extract object name from .versions directory name (remove .versions suffix)
|
||||
baseObjectName := strings.TrimSuffix(versionsDir, ".versions")
|
||||
|
||||
// Construct full object path relative to bucket
|
||||
// dir is something like "/buckets/sea-test-1/Veeam/Backup/vbr/Config"
|
||||
// we need to get the path relative to bucket: "Veeam/Backup/vbr/Config/Owner"
|
||||
bucketPath := strings.TrimPrefix(dir, s3a.option.BucketsPath+"/")
|
||||
bucketName := strings.Split(bucketPath, "/")[0]
|
||||
|
||||
// Remove bucket name from path to get directory within bucket
|
||||
bucketRelativePath := strings.Join(strings.Split(bucketPath, "/")[1:], "/")
|
||||
|
||||
var fullObjectPath string
|
||||
if bucketRelativePath == "" {
|
||||
// Object is at bucket root
|
||||
fullObjectPath = baseObjectName
|
||||
} else {
|
||||
// Object is in subdirectory
|
||||
fullObjectPath = bucketRelativePath + "/" + baseObjectName
|
||||
}
|
||||
|
||||
glog.V(4).Infof("Processing versioned object: baseObjectName=%s, bucketRelativePath=%s, fullObjectPath=%s",
|
||||
baseObjectName, bucketRelativePath, fullObjectPath)
|
||||
|
||||
// Get the latest version information for this object
|
||||
if latestVersionEntry, latestVersionErr := s3a.getLatestVersionEntryForListOperation(bucketName, fullObjectPath); latestVersionErr == nil {
|
||||
glog.V(4).Infof("Creating logical entry for versioned object: %s", fullObjectPath)
|
||||
eachEntryFn(dir, latestVersionEntry)
|
||||
} else {
|
||||
glog.V(4).Infof("Failed to get latest version for %s: %v", fullObjectPath, latestVersionErr)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -617,32 +513,3 @@ func (s3a *S3ApiServer) ensureDirectoryAllEmpty(filerClient filer_pb.SeaweedFile
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// getLatestVersionEntryForListOperation gets the latest version of an object and creates a logical entry for list operations
|
||||
// This is used to show versioned objects as logical object names in regular list operations
|
||||
func (s3a *S3ApiServer) getLatestVersionEntryForListOperation(bucket, object string) (*filer_pb.Entry, error) {
|
||||
// Get the latest version entry
|
||||
latestVersionEntry, err := s3a.getLatestObjectVersion(bucket, object)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get latest version: %w", err)
|
||||
}
|
||||
|
||||
// Check if this is a delete marker (should not be shown in regular list)
|
||||
if latestVersionEntry.Extended != nil {
|
||||
if deleteMarker, exists := latestVersionEntry.Extended[s3_constants.ExtDeleteMarkerKey]; exists && string(deleteMarker) == "true" {
|
||||
return nil, fmt.Errorf("latest version is a delete marker")
|
||||
}
|
||||
}
|
||||
|
||||
// Create a logical entry that appears to be stored at the object path (not the versioned path)
|
||||
// This allows the list operation to show the logical object name while preserving all metadata
|
||||
logicalEntry := &filer_pb.Entry{
|
||||
Name: strings.TrimPrefix(object, "/"),
|
||||
IsDirectory: false,
|
||||
Attributes: latestVersionEntry.Attributes,
|
||||
Extended: latestVersionEntry.Extended,
|
||||
Chunks: latestVersionEntry.Chunks,
|
||||
}
|
||||
|
||||
return logicalEntry, nil
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package s3api
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -42,7 +41,7 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
|
||||
// Check if versioning is enabled for the bucket (needed for object lock)
|
||||
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
@@ -112,7 +111,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
response, errCode := s3a.completeMultipartUpload(r, &s3.CompleteMultipartUploadInput{
|
||||
response, errCode := s3a.completeMultipartUpload(&s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: objectKey(aws.String(object)),
|
||||
UploadId: aws.String(uploadID),
|
||||
|
||||
@@ -90,18 +90,15 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
entry.Content, _ = io.ReadAll(r.Body)
|
||||
}
|
||||
entry.Attributes.Mime = objectContentType
|
||||
|
||||
// Set object owner for directory objects (same as regular objects)
|
||||
s3a.setObjectOwnerFromRequest(r, entry)
|
||||
}); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// Get detailed versioning state for the bucket
|
||||
versioningState, err := s3a.getVersioningState(bucket)
|
||||
// Check if versioning is enabled for the bucket
|
||||
versioningEnabled, err := s3a.isVersioningEnabled(bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
@@ -110,10 +107,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
versioningEnabled := (versioningState == s3_constants.VersioningEnabled)
|
||||
versioningConfigured := (versioningState != "")
|
||||
|
||||
glog.V(1).Infof("PutObjectHandler: bucket %s, object %s, versioningState=%s", bucket, object, versioningState)
|
||||
glog.V(1).Infof("PutObjectHandler: bucket %s, object %s, versioningEnabled=%v", bucket, object, versioningEnabled)
|
||||
|
||||
// Validate object lock headers before processing
|
||||
if err := s3a.validateObjectLockHeaders(r, versioningEnabled); err != nil {
|
||||
@@ -124,7 +118,7 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
// For non-versioned buckets, check if existing object has object lock protections
|
||||
// that would prevent overwrite (PUT operations overwrite existing objects in non-versioned buckets)
|
||||
if !versioningConfigured {
|
||||
if !versioningEnabled {
|
||||
governanceBypassAllowed := s3a.evaluateGovernanceBypassRequest(r, bucket, object)
|
||||
if err := s3a.enforceObjectLockProtections(r, bucket, object, "", governanceBypassAllowed); err != nil {
|
||||
glog.V(2).Infof("PutObjectHandler: object lock permissions check failed for %s/%s: %v", bucket, object, err)
|
||||
@@ -133,8 +127,8 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
if versioningState == s3_constants.VersioningEnabled {
|
||||
// Handle enabled versioning - create new versions with real version IDs
|
||||
if versioningEnabled {
|
||||
// Handle versioned PUT
|
||||
glog.V(1).Infof("PutObjectHandler: using versioned PUT for %s/%s", bucket, object)
|
||||
versionId, etag, errCode := s3a.putVersionedObject(r, bucket, object, dataReader, objectContentType)
|
||||
if errCode != s3err.ErrNone {
|
||||
@@ -147,24 +141,10 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
w.Header().Set("x-amz-version-id", versionId)
|
||||
}
|
||||
|
||||
// Set ETag in response
|
||||
setEtag(w, etag)
|
||||
} else if versioningState == s3_constants.VersioningSuspended {
|
||||
// Handle suspended versioning - overwrite with "null" version ID but preserve existing versions
|
||||
glog.V(1).Infof("PutObjectHandler: using suspended versioning PUT for %s/%s", bucket, object)
|
||||
etag, errCode := s3a.putSuspendedVersioningObject(r, bucket, object, dataReader, objectContentType)
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Note: Suspended versioning should NOT return x-amz-version-id header according to AWS S3 spec
|
||||
// The object is stored with "null" version internally but no version header is returned
|
||||
|
||||
// Set ETag in response
|
||||
setEtag(w, etag)
|
||||
} else {
|
||||
// Handle regular PUT (never configured versioning)
|
||||
// Handle regular PUT (non-versioned)
|
||||
glog.V(1).Infof("PutObjectHandler: using regular PUT for %s/%s", bucket, object)
|
||||
uploadUrl := s3a.toFilerUrl(bucket, object)
|
||||
if objectContentType == "" {
|
||||
@@ -178,7 +158,6 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
// No version ID header for never-configured versioning
|
||||
setEtag(w, etag)
|
||||
}
|
||||
}
|
||||
@@ -216,14 +195,6 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader
|
||||
proxyReq.Header.Add(header, value)
|
||||
}
|
||||
}
|
||||
|
||||
// Set object owner header for filer to extract
|
||||
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
|
||||
if amzAccountId != "" {
|
||||
proxyReq.Header.Set(s3_constants.ExtAmzOwnerKey, amzAccountId)
|
||||
glog.V(2).Infof("putToFiler: setting owner header %s for object %s", amzAccountId, uploadUrl)
|
||||
}
|
||||
|
||||
// ensure that the Authorization header is overriding any previous
|
||||
// Authorization header which might be already present in proxyReq
|
||||
s3a.maybeAddFilerJwtAuthorization(proxyReq, true)
|
||||
@@ -255,8 +226,8 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, uploadUrl string, dataReader
|
||||
glog.Errorf("upload to filer error: %v", ret.Error)
|
||||
return "", filerErrorToS3Error(ret.Error)
|
||||
}
|
||||
|
||||
stats_collect.RecordBucketActiveTime(bucket)
|
||||
stats_collect.S3BucketTrafficReceivedBytesCounter.WithLabelValues(bucket).Add(float64(ret.Size))
|
||||
return etag, s3err.ErrNone
|
||||
}
|
||||
|
||||
@@ -301,149 +272,8 @@ func (s3a *S3ApiServer) maybeGetFilerJwtAuthorizationToken(isWrite bool) string
|
||||
return string(encodedJwt)
|
||||
}
|
||||
|
||||
// setObjectOwnerFromRequest sets the object owner metadata based on the authenticated user
|
||||
func (s3a *S3ApiServer) setObjectOwnerFromRequest(r *http.Request, entry *filer_pb.Entry) {
|
||||
amzAccountId := r.Header.Get(s3_constants.AmzAccountId)
|
||||
if amzAccountId != "" {
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
entry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(amzAccountId)
|
||||
glog.V(2).Infof("setObjectOwnerFromRequest: set object owner to %s", amzAccountId)
|
||||
}
|
||||
}
|
||||
|
||||
// putVersionedObject handles PUT operations for versioned buckets using the new layout
|
||||
// where all versions (including latest) are stored in the .versions directory
|
||||
func (s3a *S3ApiServer) putSuspendedVersioningObject(r *http.Request, bucket, object string, dataReader io.Reader, objectContentType string) (etag string, errCode s3err.ErrorCode) {
|
||||
// For suspended versioning, store as regular object (version ID "null") but preserve existing versions
|
||||
glog.V(2).Infof("putSuspendedVersioningObject: creating null version for %s/%s", bucket, object)
|
||||
|
||||
uploadUrl := s3a.toFilerUrl(bucket, object)
|
||||
if objectContentType == "" {
|
||||
dataReader = mimeDetect(r, dataReader)
|
||||
}
|
||||
|
||||
etag, errCode = s3a.putToFiler(r, uploadUrl, dataReader, "", bucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
glog.Errorf("putSuspendedVersioningObject: failed to upload object: %v", errCode)
|
||||
return "", errCode
|
||||
}
|
||||
|
||||
// Get the uploaded entry to add version metadata indicating this is "null" version
|
||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||
entry, err := s3a.getEntry(bucketDir, object)
|
||||
if err != nil {
|
||||
glog.Errorf("putSuspendedVersioningObject: failed to get object entry: %v", err)
|
||||
return "", s3err.ErrInternalError
|
||||
}
|
||||
|
||||
// Add metadata to indicate this is a "null" version for suspended versioning
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
entry.Extended[s3_constants.ExtVersionIdKey] = []byte("null")
|
||||
|
||||
// Set object owner for suspended versioning objects
|
||||
s3a.setObjectOwnerFromRequest(r, entry)
|
||||
|
||||
// Extract and store object lock metadata from request headers (if any)
|
||||
if err := s3a.extractObjectLockMetadataFromRequest(r, entry); err != nil {
|
||||
glog.Errorf("putSuspendedVersioningObject: failed to extract object lock metadata: %v", err)
|
||||
return "", s3err.ErrInvalidRequest
|
||||
}
|
||||
|
||||
// Update the entry with metadata
|
||||
err = s3a.mkFile(bucketDir, object, entry.Chunks, func(updatedEntry *filer_pb.Entry) {
|
||||
updatedEntry.Extended = entry.Extended
|
||||
updatedEntry.Attributes = entry.Attributes
|
||||
updatedEntry.Chunks = entry.Chunks
|
||||
})
|
||||
if err != nil {
|
||||
glog.Errorf("putSuspendedVersioningObject: failed to update object metadata: %v", err)
|
||||
return "", s3err.ErrInternalError
|
||||
}
|
||||
|
||||
// Update all existing versions/delete markers to set IsLatest=false since "null" is now latest
|
||||
err = s3a.updateIsLatestFlagsForSuspendedVersioning(bucket, object)
|
||||
if err != nil {
|
||||
glog.Warningf("putSuspendedVersioningObject: failed to update IsLatest flags: %v", err)
|
||||
// Don't fail the request, but log the warning
|
||||
}
|
||||
|
||||
glog.V(2).Infof("putSuspendedVersioningObject: successfully created null version for %s/%s", bucket, object)
|
||||
return etag, s3err.ErrNone
|
||||
}
|
||||
|
||||
// updateIsLatestFlagsForSuspendedVersioning sets IsLatest=false on all existing versions/delete markers
|
||||
// when a new "null" version becomes the latest during suspended versioning
|
||||
func (s3a *S3ApiServer) updateIsLatestFlagsForSuspendedVersioning(bucket, object string) error {
|
||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||
versionsObjectPath := object + ".versions"
|
||||
versionsDir := bucketDir + "/" + versionsObjectPath
|
||||
|
||||
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: updating flags for %s%s", bucket, object)
|
||||
|
||||
// Check if .versions directory exists
|
||||
_, err := s3a.getEntry(bucketDir, versionsObjectPath)
|
||||
if err != nil {
|
||||
// No .versions directory exists, nothing to update
|
||||
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: no .versions directory for %s%s", bucket, object)
|
||||
return nil
|
||||
}
|
||||
|
||||
// List all entries in .versions directory
|
||||
entries, _, err := s3a.list(versionsDir, "", "", false, 1000)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list versions directory: %v", err)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: found %d entries to update", len(entries))
|
||||
|
||||
// Update each version/delete marker to set IsLatest=false
|
||||
for _, entry := range entries {
|
||||
if entry.Extended == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if this entry has a version ID (it should be a version or delete marker)
|
||||
versionIdBytes, hasVersionId := entry.Extended[s3_constants.ExtVersionIdKey]
|
||||
if !hasVersionId {
|
||||
continue
|
||||
}
|
||||
|
||||
versionId := string(versionIdBytes)
|
||||
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: setting IsLatest=false for version %s", versionId)
|
||||
|
||||
// Update the entry to set IsLatest=false (we don't explicitly store this flag,
|
||||
// it's determined by comparison with latest version metadata)
|
||||
// We need to clear the latest version metadata from the .versions directory
|
||||
// so that our getObjectVersionList function will correctly show IsLatest=false
|
||||
}
|
||||
|
||||
// Clear the latest version metadata from .versions directory since "null" is now latest
|
||||
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
|
||||
if err == nil && versionsEntry.Extended != nil {
|
||||
// Remove latest version metadata so all versions show IsLatest=false
|
||||
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionIdKey)
|
||||
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionFileNameKey)
|
||||
|
||||
// Update the .versions directory entry
|
||||
err = s3a.mkFile(bucketDir, versionsObjectPath, versionsEntry.Chunks, func(updatedEntry *filer_pb.Entry) {
|
||||
updatedEntry.Extended = versionsEntry.Extended
|
||||
updatedEntry.Attributes = versionsEntry.Attributes
|
||||
updatedEntry.Chunks = versionsEntry.Chunks
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update .versions directory metadata: %v", err)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("updateIsLatestFlagsForSuspendedVersioning: cleared latest version metadata for %s%s", bucket, object)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) putVersionedObject(r *http.Request, bucket, object string, dataReader io.Reader, objectContentType string) (versionId string, etag string, errCode s3err.ErrorCode) {
|
||||
// Generate version ID
|
||||
versionId = generateVersionId()
|
||||
@@ -492,9 +322,6 @@ func (s3a *S3ApiServer) putVersionedObject(r *http.Request, bucket, object strin
|
||||
}
|
||||
versionEntry.Extended[s3_constants.ExtETagKey] = []byte(etag)
|
||||
|
||||
// Set object owner for versioned objects
|
||||
s3a.setObjectOwnerFromRequest(r, versionEntry)
|
||||
|
||||
// Extract and store object lock metadata from request headers
|
||||
if err := s3a.extractObjectLockMetadataFromRequest(r, versionEntry); err != nil {
|
||||
glog.Errorf("putVersionedObject: failed to extract object lock metadata: %v", err)
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// GetObjectAclHandler Get object ACL
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html
|
||||
func (s3a *S3ApiServer) GetObjectAclHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
}
|
||||
|
||||
// PutObjectAclHandler Put object ACL
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html
|
||||
func (s3a *S3ApiServer) PutObjectAclHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package s3api
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
@@ -19,31 +20,18 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
)
|
||||
|
||||
// S3ListObjectVersionsResult - Custom struct for S3 list-object-versions response
|
||||
// This avoids conflicts with the XSD generated ListVersionsResult struct
|
||||
// and ensures proper separation of versions and delete markers into arrays
|
||||
type S3ListObjectVersionsResult struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult"`
|
||||
|
||||
Name string `xml:"Name"`
|
||||
Prefix string `xml:"Prefix,omitempty"`
|
||||
KeyMarker string `xml:"KeyMarker,omitempty"`
|
||||
VersionIdMarker string `xml:"VersionIdMarker,omitempty"`
|
||||
NextKeyMarker string `xml:"NextKeyMarker,omitempty"`
|
||||
NextVersionIdMarker string `xml:"NextVersionIdMarker,omitempty"`
|
||||
MaxKeys int `xml:"MaxKeys"`
|
||||
Delimiter string `xml:"Delimiter,omitempty"`
|
||||
IsTruncated bool `xml:"IsTruncated"`
|
||||
|
||||
// These are the critical fields - arrays instead of single elements
|
||||
Versions []VersionEntry `xml:"Version,omitempty"` // Array for versions
|
||||
DeleteMarkers []DeleteMarkerEntry `xml:"DeleteMarker,omitempty"` // Array for delete markers
|
||||
|
||||
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
|
||||
EncodingType string `xml:"EncodingType,omitempty"`
|
||||
// ObjectVersion represents a version of an S3 object
|
||||
type ObjectVersion struct {
|
||||
VersionId string
|
||||
IsLatest bool
|
||||
IsDeleteMarker bool
|
||||
LastModified time.Time
|
||||
ETag string
|
||||
Size int64
|
||||
Entry *filer_pb.Entry
|
||||
}
|
||||
|
||||
// Original struct - keeping for compatibility but will use S3ListObjectVersionsResult for XML response
|
||||
// ListObjectVersionsResult represents the response for ListObjectVersions
|
||||
type ListObjectVersionsResult struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult"`
|
||||
Name string `xml:"Name"`
|
||||
@@ -60,37 +48,20 @@ type ListObjectVersionsResult struct {
|
||||
CommonPrefixes []PrefixEntry `xml:"CommonPrefixes,omitempty"`
|
||||
}
|
||||
|
||||
// ObjectVersion represents a version of an S3 object
|
||||
type ObjectVersion struct {
|
||||
VersionId string
|
||||
IsLatest bool
|
||||
IsDeleteMarker bool
|
||||
LastModified time.Time
|
||||
ETag string
|
||||
Size int64
|
||||
Entry *filer_pb.Entry
|
||||
}
|
||||
|
||||
// generateVersionId creates a unique version ID that preserves chronological order
|
||||
// generateVersionId creates a unique version ID
|
||||
func generateVersionId() string {
|
||||
// Use nanosecond timestamp to ensure chronological ordering
|
||||
// Format as 16-digit hex (first 16 chars of version ID)
|
||||
now := time.Now().UnixNano()
|
||||
timestampHex := fmt.Sprintf("%016x", now)
|
||||
|
||||
// Generate random 8 bytes for uniqueness (last 16 chars of version ID)
|
||||
randBytes := make([]byte, 8)
|
||||
// Generate a random 16-byte value
|
||||
randBytes := make([]byte, 16)
|
||||
if _, err := rand.Read(randBytes); err != nil {
|
||||
glog.Errorf("Failed to generate random bytes for version ID: %v", err)
|
||||
// Fallback to timestamp-only if random generation fails
|
||||
return timestampHex + "0000000000000000"
|
||||
return ""
|
||||
}
|
||||
|
||||
// Combine timestamp (16 chars) + random (16 chars) = 32 chars total
|
||||
randomHex := hex.EncodeToString(randBytes)
|
||||
versionId := timestampHex + randomHex
|
||||
// Hash with current timestamp for uniqueness
|
||||
hash := sha256.Sum256(append(randBytes, []byte(fmt.Sprintf("%d", time.Now().UnixNano()))...))
|
||||
|
||||
return versionId
|
||||
// Return first 32 characters of hex string (same length as AWS S3 version IDs)
|
||||
return hex.EncodeToString(hash[:])[:32]
|
||||
}
|
||||
|
||||
// getVersionedObjectDir returns the directory path for storing object versions
|
||||
@@ -148,23 +119,62 @@ func (s3a *S3ApiServer) createDeleteMarker(bucket, object string) (string, error
|
||||
}
|
||||
|
||||
// listObjectVersions lists all versions of an object
|
||||
func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int) (*S3ListObjectVersionsResult, error) {
|
||||
func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdMarker, delimiter string, maxKeys int) (*ListObjectVersionsResult, error) {
|
||||
var allVersions []interface{} // Can contain VersionEntry or DeleteMarkerEntry
|
||||
|
||||
// Track objects that have been processed to avoid duplicates
|
||||
processedObjects := make(map[string]bool)
|
||||
|
||||
// Track version IDs globally to prevent duplicates throughout the listing
|
||||
seenVersionIds := make(map[string]bool)
|
||||
|
||||
// Recursively find all .versions directories in the bucket
|
||||
bucketPath := path.Join(s3a.option.BucketsPath, bucket)
|
||||
err := s3a.findVersionsRecursively(bucketPath, "", &allVersions, processedObjects, seenVersionIds, bucket, prefix)
|
||||
// List all entries in bucket
|
||||
entries, _, err := s3a.list(path.Join(s3a.option.BucketsPath, bucket), prefix, keyMarker, false, uint32(maxKeys*2))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Sort by key, then by LastModified (newest first), then by VersionId for deterministic ordering
|
||||
// For each entry, check if it's a .versions directory
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDirectory {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if this is a .versions directory
|
||||
if !strings.HasSuffix(entry.Name, ".versions") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract object name from .versions directory name
|
||||
objectKey := strings.TrimSuffix(entry.Name, ".versions")
|
||||
|
||||
versions, err := s3a.getObjectVersionList(bucket, objectKey)
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to get versions for object %s: %v", objectKey, err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, version := range versions {
|
||||
if version.IsDeleteMarker {
|
||||
deleteMarker := &DeleteMarkerEntry{
|
||||
Key: objectKey,
|
||||
VersionId: version.VersionId,
|
||||
IsLatest: version.IsLatest,
|
||||
LastModified: version.LastModified,
|
||||
Owner: CanonicalUser{ID: "unknown", DisplayName: "unknown"},
|
||||
}
|
||||
allVersions = append(allVersions, deleteMarker)
|
||||
} else {
|
||||
versionEntry := &VersionEntry{
|
||||
Key: objectKey,
|
||||
VersionId: version.VersionId,
|
||||
IsLatest: version.IsLatest,
|
||||
LastModified: version.LastModified,
|
||||
ETag: version.ETag,
|
||||
Size: version.Size,
|
||||
Owner: CanonicalUser{ID: "unknown", DisplayName: "unknown"},
|
||||
StorageClass: "STANDARD",
|
||||
}
|
||||
allVersions = append(allVersions, versionEntry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by key, then by LastModified and VersionId
|
||||
sort.Slice(allVersions, func(i, j int) bool {
|
||||
var keyI, keyJ string
|
||||
var lastModifiedI, lastModifiedJ time.Time
|
||||
@@ -192,24 +202,17 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
|
||||
versionIdJ = v.VersionId
|
||||
}
|
||||
|
||||
// First sort by object key
|
||||
if keyI != keyJ {
|
||||
return keyI < keyJ
|
||||
}
|
||||
|
||||
// Then by modification time (newest first) - but use nanosecond precision for ties
|
||||
timeDiff := lastModifiedI.Sub(lastModifiedJ)
|
||||
if timeDiff.Abs() > time.Millisecond {
|
||||
if !lastModifiedI.Equal(lastModifiedJ) {
|
||||
return lastModifiedI.After(lastModifiedJ)
|
||||
}
|
||||
|
||||
// For very close timestamps (within 1ms), use version ID for deterministic ordering
|
||||
// Sort version IDs in reverse lexicographic order to maintain newest-first semantics
|
||||
return versionIdI > versionIdJ
|
||||
return versionIdI < versionIdJ
|
||||
})
|
||||
|
||||
// Build result using S3ListObjectVersionsResult to avoid conflicts with XSD structs
|
||||
result := &S3ListObjectVersionsResult{
|
||||
// Build result
|
||||
result := &ListObjectVersionsResult{
|
||||
Name: bucket,
|
||||
Prefix: prefix,
|
||||
KeyMarker: keyMarker,
|
||||
@@ -234,10 +237,6 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
|
||||
}
|
||||
}
|
||||
|
||||
// Always initialize empty slices so boto3 gets the expected fields even when empty
|
||||
result.Versions = make([]VersionEntry, 0)
|
||||
result.DeleteMarkers = make([]DeleteMarkerEntry, 0)
|
||||
|
||||
// Add versions to result
|
||||
for _, version := range allVersions {
|
||||
switch v := version.(type) {
|
||||
@@ -251,172 +250,6 @@ func (s3a *S3ApiServer) listObjectVersions(bucket, prefix, keyMarker, versionIdM
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// findVersionsRecursively searches for all .versions directories and regular files recursively
|
||||
func (s3a *S3ApiServer) findVersionsRecursively(currentPath, relativePath string, allVersions *[]interface{}, processedObjects map[string]bool, seenVersionIds map[string]bool, bucket, prefix string) error {
|
||||
// List entries in current directory
|
||||
entries, _, err := s3a.list(currentPath, "", "", false, 1000)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
entryPath := path.Join(relativePath, entry.Name)
|
||||
|
||||
// Skip if this doesn't match the prefix filter
|
||||
if prefix != "" {
|
||||
normalizedPrefix := strings.TrimPrefix(prefix, "/")
|
||||
if normalizedPrefix != "" {
|
||||
// An entry is a candidate if:
|
||||
// 1. Its path is prefixed by normalizedPrefix.
|
||||
// 2. It is a directory that is a prefix of normalizedPrefix (so we can descend into it).
|
||||
isPrefixed := strings.HasPrefix(entryPath, normalizedPrefix)
|
||||
if !isPrefixed && entry.IsDirectory {
|
||||
// For directories, also check with a trailing slash.
|
||||
isPrefixed = strings.HasPrefix(entryPath+"/", normalizedPrefix)
|
||||
}
|
||||
|
||||
canDescend := entry.IsDirectory && strings.HasPrefix(normalizedPrefix, entryPath)
|
||||
|
||||
if !isPrefixed && !canDescend {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if entry.IsDirectory {
|
||||
// Skip .uploads directory (multipart upload temporary files)
|
||||
if strings.HasPrefix(entry.Name, ".uploads") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if this is a .versions directory
|
||||
if strings.HasSuffix(entry.Name, ".versions") {
|
||||
// Extract object name from .versions directory name
|
||||
objectKey := strings.TrimSuffix(entryPath, ".versions")
|
||||
processedObjects[objectKey] = true
|
||||
|
||||
glog.V(2).Infof("findVersionsRecursively: found .versions directory for object %s", objectKey)
|
||||
|
||||
versions, err := s3a.getObjectVersionList(bucket, objectKey)
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to get versions for object %s: %v", objectKey, err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, version := range versions {
|
||||
// Check for duplicate version IDs and skip if already seen
|
||||
versionKey := objectKey + ":" + version.VersionId
|
||||
if seenVersionIds[versionKey] {
|
||||
glog.Warningf("findVersionsRecursively: duplicate version %s for object %s detected, skipping", version.VersionId, objectKey)
|
||||
continue
|
||||
}
|
||||
seenVersionIds[versionKey] = true
|
||||
|
||||
if version.IsDeleteMarker {
|
||||
deleteMarker := &DeleteMarkerEntry{
|
||||
Key: objectKey,
|
||||
VersionId: version.VersionId,
|
||||
IsLatest: version.IsLatest,
|
||||
LastModified: version.LastModified,
|
||||
Owner: s3a.getObjectOwnerFromVersion(version, bucket, objectKey),
|
||||
}
|
||||
*allVersions = append(*allVersions, deleteMarker)
|
||||
} else {
|
||||
versionEntry := &VersionEntry{
|
||||
Key: objectKey,
|
||||
VersionId: version.VersionId,
|
||||
IsLatest: version.IsLatest,
|
||||
LastModified: version.LastModified,
|
||||
ETag: version.ETag,
|
||||
Size: version.Size,
|
||||
Owner: s3a.getObjectOwnerFromVersion(version, bucket, objectKey),
|
||||
StorageClass: "STANDARD",
|
||||
}
|
||||
*allVersions = append(*allVersions, versionEntry)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// This is a regular directory - check if it's an explicit S3 directory object
|
||||
// Only include directories that were explicitly created via S3 API (have FolderMimeType)
|
||||
// This excludes implicit directories created when uploading files like "test1/a"
|
||||
if entry.Attributes.Mime == s3_constants.FolderMimeType {
|
||||
directoryKey := entryPath
|
||||
if !strings.HasSuffix(directoryKey, "/") {
|
||||
directoryKey += "/"
|
||||
}
|
||||
|
||||
// Add directory as a version entry with VersionId "null" (following S3/Minio behavior)
|
||||
glog.V(2).Infof("findVersionsRecursively: found explicit S3 directory %s", directoryKey)
|
||||
|
||||
// Calculate ETag for empty directory
|
||||
directoryETag := "\"d41d8cd98f00b204e9800998ecf8427e\""
|
||||
|
||||
versionEntry := &VersionEntry{
|
||||
Key: directoryKey,
|
||||
VersionId: "null",
|
||||
IsLatest: true,
|
||||
LastModified: time.Unix(entry.Attributes.Mtime, 0),
|
||||
ETag: directoryETag,
|
||||
Size: 0, // Directories have size 0
|
||||
Owner: s3a.getObjectOwnerFromEntry(entry),
|
||||
StorageClass: "STANDARD",
|
||||
}
|
||||
*allVersions = append(*allVersions, versionEntry)
|
||||
}
|
||||
|
||||
// Recursively search subdirectories (regardless of whether they're explicit or implicit)
|
||||
fullPath := path.Join(currentPath, entry.Name)
|
||||
err := s3a.findVersionsRecursively(fullPath, entryPath, allVersions, processedObjects, seenVersionIds, bucket, prefix)
|
||||
if err != nil {
|
||||
glog.Warningf("Error searching subdirectory %s: %v", entryPath, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// This is a regular file - check if it's a pre-versioning object
|
||||
objectKey := entryPath
|
||||
|
||||
// Skip if this object already has a .versions directory (already processed)
|
||||
if processedObjects[objectKey] {
|
||||
continue
|
||||
}
|
||||
|
||||
// This is a pre-versioning object - treat it as a version with VersionId="null"
|
||||
glog.V(2).Infof("findVersionsRecursively: found pre-versioning object %s", objectKey)
|
||||
|
||||
// Check if this null version should be marked as latest
|
||||
// It's only latest if there's no .versions directory OR no latest version metadata
|
||||
isLatest := true
|
||||
versionsObjectPath := objectKey + ".versions"
|
||||
if versionsEntry, err := s3a.getEntry(currentPath, versionsObjectPath); err == nil {
|
||||
// .versions directory exists, check if there's latest version metadata
|
||||
if versionsEntry.Extended != nil {
|
||||
if _, hasLatest := versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey]; hasLatest {
|
||||
// There is a latest version in the .versions directory, so null is not latest
|
||||
isLatest = false
|
||||
glog.V(2).Infof("findVersionsRecursively: null version for %s is not latest due to versioned objects", objectKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
etag := s3a.calculateETagFromChunks(entry.Chunks)
|
||||
versionEntry := &VersionEntry{
|
||||
Key: objectKey,
|
||||
VersionId: "null",
|
||||
IsLatest: isLatest,
|
||||
LastModified: time.Unix(entry.Attributes.Mtime, 0),
|
||||
ETag: etag,
|
||||
Size: int64(entry.Attributes.FileSize),
|
||||
Owner: s3a.getObjectOwnerFromEntry(entry),
|
||||
StorageClass: "STANDARD",
|
||||
}
|
||||
*allVersions = append(*allVersions, versionEntry)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getObjectVersionList returns all versions of a specific object
|
||||
func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVersion, error) {
|
||||
var versions []*ObjectVersion
|
||||
@@ -454,9 +287,6 @@ func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVe
|
||||
|
||||
glog.V(2).Infof("getObjectVersionList: found %d entries in versions directory", len(entries))
|
||||
|
||||
// Use a map to detect and prevent duplicate version IDs
|
||||
seenVersionIds := make(map[string]bool)
|
||||
|
||||
for i, entry := range entries {
|
||||
if entry.Extended == nil {
|
||||
glog.V(2).Infof("getObjectVersionList: entry %d has no Extended metadata, skipping", i)
|
||||
@@ -471,13 +301,6 @@ func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVe
|
||||
|
||||
versionId := string(versionIdBytes)
|
||||
|
||||
// Check for duplicate version IDs and skip if already seen
|
||||
if seenVersionIds[versionId] {
|
||||
glog.Warningf("getObjectVersionList: duplicate version ID %s detected for object %s/%s, skipping", versionId, bucket, object)
|
||||
continue
|
||||
}
|
||||
seenVersionIds[versionId] = true
|
||||
|
||||
// Check if this version is the latest by comparing with directory metadata
|
||||
isLatest := (versionId == latestVersionId)
|
||||
|
||||
@@ -508,9 +331,12 @@ func (s3a *S3ApiServer) getObjectVersionList(bucket, object string) ([]*ObjectVe
|
||||
versions = append(versions, version)
|
||||
}
|
||||
|
||||
// Don't sort here - let the main listObjectVersions function handle sorting consistently
|
||||
// Sort by modification time (newest first)
|
||||
sort.Slice(versions, func(i, j int) bool {
|
||||
return versions[i].LastModified.After(versions[j].LastModified)
|
||||
})
|
||||
|
||||
glog.V(2).Infof("getObjectVersionList: returning %d total versions for %s/%s (after deduplication from %d entries)", len(versions), bucket, object, len(entries))
|
||||
glog.V(2).Infof("getObjectVersionList: returning %d total versions for %s/%s", len(versions), bucket, object)
|
||||
for i, version := range versions {
|
||||
glog.V(2).Infof("getObjectVersionList: version %d: %s (isLatest=%v, isDeleteMarker=%v)", i, version.VersionId, version.IsLatest, version.IsDeleteMarker)
|
||||
}
|
||||
@@ -540,16 +366,6 @@ func (s3a *S3ApiServer) getSpecificObjectVersion(bucket, object, versionId strin
|
||||
return s3a.getEntry(path.Join(s3a.option.BucketsPath, bucket), strings.TrimPrefix(object, "/"))
|
||||
}
|
||||
|
||||
if versionId == "null" {
|
||||
// "null" version ID refers to pre-versioning objects stored as regular files
|
||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||
entry, err := s3a.getEntry(bucketDir, object)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("null version object %s not found: %v", object, err)
|
||||
}
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
// Get specific version from .versions directory
|
||||
versionsDir := s3a.getVersionedObjectDir(bucket, object)
|
||||
versionFile := s3a.getVersionFileName(versionId)
|
||||
@@ -568,153 +384,25 @@ func (s3a *S3ApiServer) deleteSpecificObjectVersion(bucket, object, versionId st
|
||||
return fmt.Errorf("version ID is required for version-specific deletion")
|
||||
}
|
||||
|
||||
if versionId == "null" {
|
||||
// Delete "null" version (pre-versioning object stored as regular file)
|
||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||
cleanObject := strings.TrimPrefix(object, "/")
|
||||
|
||||
// Check if the object exists
|
||||
_, err := s3a.getEntry(bucketDir, cleanObject)
|
||||
if err != nil {
|
||||
// Object doesn't exist - this is OK for delete operations (idempotent)
|
||||
glog.V(2).Infof("deleteSpecificObjectVersion: null version object %s already deleted or doesn't exist", cleanObject)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete the regular file
|
||||
deleteErr := s3a.rm(bucketDir, cleanObject, true, false)
|
||||
if deleteErr != nil {
|
||||
// Check if file was already deleted by another process
|
||||
if _, checkErr := s3a.getEntry(bucketDir, cleanObject); checkErr != nil {
|
||||
// File doesn't exist anymore, deletion was successful
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to delete null version %s: %v", cleanObject, deleteErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
versionsDir := s3a.getVersionedObjectDir(bucket, object)
|
||||
versionFile := s3a.getVersionFileName(versionId)
|
||||
|
||||
// Check if this is the latest version before attempting deletion (for potential metadata update)
|
||||
versionsEntry, dirErr := s3a.getEntry(path.Join(s3a.option.BucketsPath, bucket), object+".versions")
|
||||
isLatestVersion := false
|
||||
if dirErr == nil && versionsEntry.Extended != nil {
|
||||
if latestVersionIdBytes, hasLatest := versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey]; hasLatest {
|
||||
isLatestVersion = (string(latestVersionIdBytes) == versionId)
|
||||
}
|
||||
// Delete the specific version from .versions directory
|
||||
_, err := s3a.getEntry(versionsDir, versionFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("version %s not found: %v", versionId, err)
|
||||
}
|
||||
|
||||
// Attempt to delete the version file
|
||||
// Note: We don't check if the file exists first to avoid race conditions
|
||||
// The deletion operation should be idempotent
|
||||
// Version exists, delete it
|
||||
deleteErr := s3a.rm(versionsDir, versionFile, true, false)
|
||||
if deleteErr != nil {
|
||||
// Check if file was already deleted by another process (race condition handling)
|
||||
// Check if file was already deleted by another process
|
||||
if _, checkErr := s3a.getEntry(versionsDir, versionFile); checkErr != nil {
|
||||
// File doesn't exist anymore, deletion was successful (another thread deleted it)
|
||||
glog.V(2).Infof("deleteSpecificObjectVersion: version %s for %s%s already deleted by another process", versionId, bucket, object)
|
||||
// File doesn't exist anymore, deletion was successful
|
||||
return nil
|
||||
}
|
||||
// File still exists but deletion failed for another reason
|
||||
return fmt.Errorf("failed to delete version %s: %v", versionId, deleteErr)
|
||||
}
|
||||
|
||||
// If we deleted the latest version, update the .versions directory metadata to point to the new latest
|
||||
if isLatestVersion {
|
||||
err := s3a.updateLatestVersionAfterDeletion(bucket, object)
|
||||
if err != nil {
|
||||
glog.Warningf("deleteSpecificObjectVersion: failed to update latest version after deletion: %v", err)
|
||||
// Don't return error since the deletion was successful
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateLatestVersionAfterDeletion finds the new latest version after deleting the current latest
|
||||
func (s3a *S3ApiServer) updateLatestVersionAfterDeletion(bucket, object string) error {
|
||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||
cleanObject := strings.TrimPrefix(object, "/")
|
||||
versionsObjectPath := cleanObject + ".versions"
|
||||
versionsDir := bucketDir + "/" + versionsObjectPath
|
||||
|
||||
glog.V(1).Infof("updateLatestVersionAfterDeletion: updating latest version for %s/%s, listing %s", bucket, object, versionsDir)
|
||||
|
||||
// List all remaining version files in the .versions directory
|
||||
entries, _, err := s3a.list(versionsDir, "", "", false, 1000)
|
||||
if err != nil {
|
||||
glog.Errorf("updateLatestVersionAfterDeletion: failed to list versions in %s: %v", versionsDir, err)
|
||||
return fmt.Errorf("failed to list versions: %v", err)
|
||||
}
|
||||
|
||||
glog.V(1).Infof("updateLatestVersionAfterDeletion: found %d entries in %s", len(entries), versionsDir)
|
||||
|
||||
// Find the most recent remaining version (latest timestamp in version ID)
|
||||
var latestVersionId string
|
||||
var latestVersionFileName string
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.Extended == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
versionIdBytes, hasVersionId := entry.Extended[s3_constants.ExtVersionIdKey]
|
||||
if !hasVersionId {
|
||||
continue
|
||||
}
|
||||
|
||||
versionId := string(versionIdBytes)
|
||||
|
||||
// Skip delete markers when finding latest content version
|
||||
isDeleteMarkerBytes, _ := entry.Extended[s3_constants.ExtDeleteMarkerKey]
|
||||
if string(isDeleteMarkerBytes) == "true" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Compare version IDs chronologically (our version IDs start with timestamp)
|
||||
if latestVersionId == "" || versionId > latestVersionId {
|
||||
glog.V(1).Infof("updateLatestVersionAfterDeletion: found newer version %s (file: %s)", versionId, entry.Name)
|
||||
latestVersionId = versionId
|
||||
latestVersionFileName = entry.Name
|
||||
} else {
|
||||
glog.V(1).Infof("updateLatestVersionAfterDeletion: skipping older version %s", versionId)
|
||||
}
|
||||
}
|
||||
|
||||
// Update the .versions directory metadata
|
||||
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get .versions directory: %v", err)
|
||||
}
|
||||
|
||||
if versionsEntry.Extended == nil {
|
||||
versionsEntry.Extended = make(map[string][]byte)
|
||||
}
|
||||
|
||||
if latestVersionId != "" {
|
||||
// Update metadata to point to new latest version
|
||||
versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey] = []byte(latestVersionId)
|
||||
versionsEntry.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte(latestVersionFileName)
|
||||
glog.V(2).Infof("updateLatestVersionAfterDeletion: new latest version for %s/%s is %s", bucket, object, latestVersionId)
|
||||
} else {
|
||||
// No versions left, remove latest version metadata
|
||||
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionIdKey)
|
||||
delete(versionsEntry.Extended, s3_constants.ExtLatestVersionFileNameKey)
|
||||
glog.V(2).Infof("updateLatestVersionAfterDeletion: no versions left for %s/%s", bucket, object)
|
||||
}
|
||||
|
||||
// Update the .versions directory entry
|
||||
err = s3a.mkFile(bucketDir, versionsObjectPath, versionsEntry.Chunks, func(updatedEntry *filer_pb.Entry) {
|
||||
updatedEntry.Extended = versionsEntry.Extended
|
||||
updatedEntry.Attributes = versionsEntry.Attributes
|
||||
updatedEntry.Chunks = versionsEntry.Chunks
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update .versions directory metadata: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -731,8 +419,7 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
|
||||
|
||||
// Parse query parameters
|
||||
query := r.URL.Query()
|
||||
originalPrefix := query.Get("prefix") // Keep original prefix for response
|
||||
prefix := originalPrefix // Use for internal processing
|
||||
prefix := query.Get("prefix")
|
||||
if prefix != "" && !strings.HasPrefix(prefix, "/") {
|
||||
prefix = "/" + prefix
|
||||
}
|
||||
@@ -757,9 +444,6 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http
|
||||
return
|
||||
}
|
||||
|
||||
// Set the original prefix in the response (not the normalized internal prefix)
|
||||
result.Prefix = originalPrefix
|
||||
|
||||
writeSuccessResponseXML(w, r, result)
|
||||
}
|
||||
|
||||
@@ -771,50 +455,19 @@ func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb
|
||||
// Get the .versions directory entry to read latest version metadata
|
||||
versionsEntry, err := s3a.getEntry(bucketDir, versionsObjectPath)
|
||||
if err != nil {
|
||||
// .versions directory doesn't exist - this can happen for objects that existed
|
||||
// before versioning was enabled on the bucket. Fall back to checking for a
|
||||
// regular (non-versioned) object file.
|
||||
glog.V(2).Infof("getLatestObjectVersion: no .versions directory for %s%s, checking for pre-versioning object", bucket, object)
|
||||
|
||||
regularEntry, regularErr := s3a.getEntry(bucketDir, object)
|
||||
if regularErr != nil {
|
||||
return nil, fmt.Errorf("failed to get %s%s .versions directory and no regular object found: %w", bucket, object, err)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("getLatestObjectVersion: found pre-versioning object for %s/%s", bucket, object)
|
||||
return regularEntry, nil
|
||||
return nil, fmt.Errorf("failed to get .versions directory: %w", err)
|
||||
}
|
||||
|
||||
// Check if directory has latest version metadata
|
||||
if versionsEntry.Extended == nil {
|
||||
// No metadata means all versioned objects have been deleted.
|
||||
// Fall back to checking for a pre-versioning object.
|
||||
glog.V(2).Infof("getLatestObjectVersion: no Extended metadata in .versions directory for %s%s, checking for pre-versioning object", bucket, object)
|
||||
|
||||
regularEntry, regularErr := s3a.getEntry(bucketDir, object)
|
||||
if regularErr != nil {
|
||||
return nil, fmt.Errorf("no version metadata in .versions directory and no regular object found for %s%s", bucket, object)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("getLatestObjectVersion: found pre-versioning object for %s%s (no Extended metadata case)", bucket, object)
|
||||
return regularEntry, nil
|
||||
return nil, fmt.Errorf("no version metadata found in .versions directory for %s/%s", bucket, object)
|
||||
}
|
||||
|
||||
latestVersionIdBytes, hasLatestVersionId := versionsEntry.Extended[s3_constants.ExtLatestVersionIdKey]
|
||||
latestVersionFileBytes, hasLatestVersionFile := versionsEntry.Extended[s3_constants.ExtLatestVersionFileNameKey]
|
||||
|
||||
if !hasLatestVersionId || !hasLatestVersionFile {
|
||||
// No version metadata means all versioned objects have been deleted.
|
||||
// Fall back to checking for a pre-versioning object.
|
||||
glog.V(2).Infof("getLatestObjectVersion: no version metadata in .versions directory for %s/%s, checking for pre-versioning object", bucket, object)
|
||||
|
||||
regularEntry, regularErr := s3a.getEntry(bucketDir, object)
|
||||
if regularErr != nil {
|
||||
return nil, fmt.Errorf("no version metadata in .versions directory and no regular object found for %s%s", bucket, object)
|
||||
}
|
||||
|
||||
glog.V(2).Infof("getLatestObjectVersion: found pre-versioning object for %s%s after version deletion", bucket, object)
|
||||
return regularEntry, nil
|
||||
return nil, fmt.Errorf("incomplete latest version metadata in .versions directory for %s/%s", bucket, object)
|
||||
}
|
||||
|
||||
latestVersionId := string(latestVersionIdBytes)
|
||||
@@ -831,55 +484,3 @@ func (s3a *S3ApiServer) getLatestObjectVersion(bucket, object string) (*filer_pb
|
||||
|
||||
return latestVersionEntry, nil
|
||||
}
|
||||
|
||||
// getObjectOwnerFromVersion extracts object owner information from version entry metadata
|
||||
func (s3a *S3ApiServer) getObjectOwnerFromVersion(version *ObjectVersion, bucket, objectKey string) CanonicalUser {
|
||||
// First try to get owner from the version entry itself
|
||||
if version.Entry != nil && version.Entry.Extended != nil {
|
||||
if ownerBytes, exists := version.Entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
|
||||
ownerId := string(ownerBytes)
|
||||
ownerDisplayName := s3a.iam.GetAccountNameById(ownerId)
|
||||
return CanonicalUser{ID: ownerId, DisplayName: ownerDisplayName}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: try to get owner from the current version of the object
|
||||
// This handles cases where older versions might not have owner metadata
|
||||
if version.VersionId == "null" {
|
||||
// For null version, check the regular object file
|
||||
bucketDir := s3a.option.BucketsPath + "/" + bucket
|
||||
if entry, err := s3a.getEntry(bucketDir, objectKey); err == nil && entry.Extended != nil {
|
||||
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
|
||||
ownerId := string(ownerBytes)
|
||||
ownerDisplayName := s3a.iam.GetAccountNameById(ownerId)
|
||||
return CanonicalUser{ID: ownerId, DisplayName: ownerDisplayName}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// For versioned objects, try to get from latest version metadata
|
||||
if latestVersion, err := s3a.getLatestObjectVersion(bucket, objectKey); err == nil && latestVersion.Extended != nil {
|
||||
if ownerBytes, exists := latestVersion.Extended[s3_constants.ExtAmzOwnerKey]; exists {
|
||||
ownerId := string(ownerBytes)
|
||||
ownerDisplayName := s3a.iam.GetAccountNameById(ownerId)
|
||||
return CanonicalUser{ID: ownerId, DisplayName: ownerDisplayName}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ultimate fallback: return anonymous if no owner found
|
||||
return CanonicalUser{ID: s3_constants.AccountAnonymousId, DisplayName: "anonymous"}
|
||||
}
|
||||
|
||||
// getObjectOwnerFromEntry extracts object owner information from a file entry
|
||||
func (s3a *S3ApiServer) getObjectOwnerFromEntry(entry *filer_pb.Entry) CanonicalUser {
|
||||
if entry != nil && entry.Extended != nil {
|
||||
if ownerBytes, exists := entry.Extended[s3_constants.ExtAmzOwnerKey]; exists {
|
||||
ownerId := string(ownerBytes)
|
||||
ownerDisplayName := s3a.iam.GetAccountNameById(ownerId)
|
||||
return CanonicalUser{ID: ownerId, DisplayName: ownerDisplayName}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: return anonymous if no owner found
|
||||
return CanonicalUser{ID: s3_constants.AccountAnonymousId, DisplayName: "anonymous"}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
type FullPath string
|
||||
|
||||
func NewFullPath(dir, name string) FullPath {
|
||||
name = strings.TrimSuffix(name, "/")
|
||||
return FullPath(dir).Child(name)
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
var (
|
||||
MAJOR_VERSION = int32(3)
|
||||
MINOR_VERSION = int32(95)
|
||||
MINOR_VERSION = int32(94)
|
||||
VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION)
|
||||
VERSION = util.SizeLimit + " " + VERSION_NUMBER
|
||||
COMMIT = ""
|
||||
|
||||
Reference in New Issue
Block a user