mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-01 04:05:54 +00:00
s3: stop treating a directory marker as a versioned object (#10573)
* s3: delete a directory marker instead of versioning it The key "dir/" is stored as the filer directory itself, so a delete marker cannot stand in for it without hiding the children underneath, and its history has to sit inside the directory it describes, where listings keep meeting it. Delete it the way an unversioned bucket already does: remove the directory when nothing is left under it, demote it to a plain directory when children remain, and drop a history an older build recorded for it. * s3: stop resolving directory markers through a version history Nothing records one for them any more, so the lookups that read it are dead weight - and the one in the listing was a filer round trip per directory marker returned, which for a bucket that keeps a marker per directory is the whole listing cost. A listing reads what a directory stands for straight off the entry it already has; a unit test pins that N markers cost one ListEntries rather than N+1. The guard that keeps a history left inside a directory by an older build from surfacing as a key named after it stays. * s3: do not let deleting "dir/" destroy the object at "dir" Writing under an existing object turns that object's entry into a directory while it keeps its data, so the keys "m2" and "m2/" end up sharing one entry. Stripping the entry to delete "m2/" therefore wiped the object at "m2" - a different key, and in a versioned bucket one no delete marker records. Leave a directory holding uploaded data alone; "m2/" does not name it. * s3: make the directory-marker delete fail closed and take the write lock The guard that spares a promoted file only fired when the entry read succeeded, so a transient filer error fell through to the delete and could destroy the object at "dir" anyway. Fail the request instead, take the object write lock so the entry cannot change between the check and the delete, and report a stale history that cannot be removed rather than leaving it to keep naming the key in ListObjectVersions. * s3: check If-Match inside the directory-marker delete lock The lock belongs to the caller: taking it inside the delete nested it under the batch handler's own lock, and since every lock from a gateway shares one owner the inner release would have freed it while the outer caller still assumed it held it. Both callers now own the lock, the single-object path re-checks If-Match inside it the way the other delete paths do, and a batch delete of a trailing-slash key in an unversioned bucket goes through the same marker path instead of the raw delete. A history lookup that fails now fails the delete.
This commit is contained in:
Chris Lu
@@ -2,6 +2,7 @@ package s3api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
@@ -12,8 +13,8 @@ import (
|
||||
)
|
||||
|
||||
// TestDeletedDirectoryMarkerDisappears covers the rclone directory_markers flow: a key
|
||||
// created with PutObject on "m2/" is deleted, and every current-version surface has to
|
||||
// agree it is gone even though the filer directory that carries it survives.
|
||||
// created with PutObject on "m2/" is deleted, and stops being a key everywhere. The
|
||||
// directory that carried it goes with it once nothing is left underneath.
|
||||
func TestDeletedDirectoryMarkerDisappears(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
@@ -24,7 +25,6 @@ func TestDeletedDirectoryMarkerDisappears(t *testing.T) {
|
||||
|
||||
putObject(t, client, bucketName, "m2/", "")
|
||||
putObject(t, client, bucketName, "m2/f.txt", "hi")
|
||||
|
||||
assert.Equal(t, []string{"m2/", "m2/f.txt"}, listKeys(t, client, bucketName, ""))
|
||||
|
||||
deleteKey(t, client, bucketName, "m2/f.txt")
|
||||
@@ -32,39 +32,22 @@ func TestDeletedDirectoryMarkerDisappears(t *testing.T) {
|
||||
|
||||
assert.Empty(t, listKeys(t, client, bucketName, ""), "the deleted marker is not a key")
|
||||
assert.Empty(t, listPrefixes(t, client, bucketName, ""), "and it names no prefix")
|
||||
assert.Empty(t, listKeys(t, client, bucketName, "m2/"), "prefix=m2/ answers empty")
|
||||
|
||||
_, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2/"),
|
||||
})
|
||||
require.Error(t, err, "HEAD on a deleted directory marker must not answer 200")
|
||||
_, err = client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2/"),
|
||||
})
|
||||
requireAPIError(t, err, "NoSuchKey")
|
||||
|
||||
// The key appears once in the version listing, as its delete marker.
|
||||
// The file's own version history is untouched by deleting the directory key.
|
||||
versions, err := client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String("m2/"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
latest := 0
|
||||
for _, v := range versions.Versions {
|
||||
if *v.Key == "m2/" && *v.IsLatest {
|
||||
latest++
|
||||
}
|
||||
assert.NotEqual(t, "m2/", *v.Key, "a directory marker is not a versioned object")
|
||||
}
|
||||
for _, m := range versions.DeleteMarkers {
|
||||
if *m.Key == "m2/" && *m.IsLatest {
|
||||
latest++
|
||||
}
|
||||
assert.NotEqual(t, "m2/", *m.Key, "deleting one writes no delete marker")
|
||||
}
|
||||
assert.Equal(t, 1, latest, "exactly one version of m2/ is the latest")
|
||||
assert.Len(t, versions.Versions, 1, "m2/f.txt keeps its version")
|
||||
assert.Len(t, versions.DeleteMarkers, 1, "and its delete marker")
|
||||
|
||||
// Re-creating the marker retires the delete marker and keeps the history.
|
||||
// Re-creating the marker brings the key back.
|
||||
putObject(t, client, bucketName, "m2/", "")
|
||||
assert.Equal(t, []string{"m2/"}, listKeys(t, client, bucketName, ""))
|
||||
_, err = client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
@@ -72,18 +55,6 @@ func TestDeletedDirectoryMarkerDisappears(t *testing.T) {
|
||||
Key: aws.String("m2/"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
versions, err = client.ListObjectVersions(context.TODO(), &s3.ListObjectVersionsInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Prefix: aws.String("m2/"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.NotEmpty(t, versions.DeleteMarkers, "the delete marker stays in the history")
|
||||
for _, m := range versions.DeleteMarkers {
|
||||
if *m.Key == "m2/" {
|
||||
assert.False(t, *m.IsLatest, "the delete marker is no longer current")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeletedDirectoryMarkerKeepsItsSubtree pins the boundary: deleting the key "m2/"
|
||||
@@ -101,29 +72,102 @@ func TestDeletedDirectoryMarkerKeepsItsSubtree(t *testing.T) {
|
||||
|
||||
deleteKey(t, client, bucketName, "m2/")
|
||||
|
||||
assert.Equal(t, []string{"m2/keep.txt"}, listKeys(t, client, bucketName, ""))
|
||||
assert.Equal(t, []string{"m2/keep.txt"}, listKeys(t, client, bucketName, ""), "the marker key is gone")
|
||||
assert.Equal(t, []string{"m2/"}, listPrefixes(t, client, bucketName, ""),
|
||||
"a live child keeps the prefix even though the marker key is gone")
|
||||
"a live child keeps the prefix")
|
||||
assert.Equal(t, []string{"m2/keep.txt"}, listKeys(t, client, bucketName, "m2/"))
|
||||
|
||||
// The surviving object is still readable through the prefix that no longer has a key.
|
||||
got, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2/keep.txt"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, got.Body.Close())
|
||||
}
|
||||
|
||||
// TestDirectoryMarkerSurvivesWithoutVersioning guards the unversioned path, which has no
|
||||
// history to consult and must keep answering as before.
|
||||
func TestDirectoryMarkerSurvivesWithoutVersioning(t *testing.T) {
|
||||
// TestDeletedDirectoryMarkerIsGoneFromReads checks the read surfaces once nothing is
|
||||
// left under the deleted key: the directory goes with it, so the path is not there.
|
||||
func TestDeletedDirectoryMarkerIsGoneFromReads(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
bucketName := getNewBucketName()
|
||||
|
||||
createBucket(t, client, bucketName)
|
||||
defer deleteBucket(t, client, bucketName)
|
||||
enableVersioning(t, client, bucketName)
|
||||
|
||||
putObject(t, client, bucketName, "m2/", "")
|
||||
assert.Equal(t, []string{"m2/"}, listKeys(t, client, bucketName, ""))
|
||||
deleteKey(t, client, bucketName, "m2/")
|
||||
|
||||
_, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2/"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Error(t, err, "HEAD on a deleted directory marker must not answer 200")
|
||||
_, err = client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2/"),
|
||||
})
|
||||
requireAPIError(t, err, "NoSuchKey")
|
||||
assert.Empty(t, listKeys(t, client, bucketName, "m2/"))
|
||||
}
|
||||
|
||||
// TestDirectoryMarkerDeleteMatchesUnversioned pins the point of the change: a directory
|
||||
// marker is deleted the same way whether or not the bucket is versioned.
|
||||
func TestDirectoryMarkerDeleteMatchesUnversioned(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
|
||||
for _, versioned := range []bool{false, true} {
|
||||
bucketName := getNewBucketName()
|
||||
createBucket(t, client, bucketName)
|
||||
if versioned {
|
||||
enableVersioning(t, client, bucketName)
|
||||
}
|
||||
|
||||
putObject(t, client, bucketName, "m/", "")
|
||||
putObject(t, client, bucketName, "m/child.txt", "x")
|
||||
deleteKey(t, client, bucketName, "m/")
|
||||
|
||||
assert.Equal(t, []string{"m/child.txt"}, listKeys(t, client, bucketName, ""),
|
||||
"versioned=%v: the marker key is gone, the child stays", versioned)
|
||||
assert.Equal(t, []string{"m/"}, listPrefixes(t, client, bucketName, ""),
|
||||
"versioned=%v: the prefix survives its live child", versioned)
|
||||
|
||||
deleteBucket(t, client, bucketName)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteDirectoryMarkerSparesAPromotedFile guards the sharp edge of storing a key
|
||||
// on a directory entry: writing under an existing object turns that object's entry into
|
||||
// a directory while it keeps its data, so "m2/" and "m2" end up on the same entry. They
|
||||
// are still different keys, and deleting one must not destroy the other.
|
||||
func TestDeleteDirectoryMarkerSparesAPromotedFile(t *testing.T) {
|
||||
client := getS3Client(t)
|
||||
|
||||
for _, versioned := range []bool{false, true} {
|
||||
bucketName := getNewBucketName()
|
||||
createBucket(t, client, bucketName)
|
||||
|
||||
putObject(t, client, bucketName, "m2", "important data")
|
||||
if versioned {
|
||||
enableVersioning(t, client, bucketName)
|
||||
}
|
||||
putObject(t, client, bucketName, "m2/child.txt", "child")
|
||||
|
||||
deleteKey(t, client, bucketName, "m2/")
|
||||
|
||||
got, err := client.GetObject(context.TODO(), &s3.GetObjectInput{
|
||||
Bucket: aws.String(bucketName),
|
||||
Key: aws.String("m2"),
|
||||
})
|
||||
require.NoError(t, err, "versioned=%v: deleting m2/ must not delete m2", versioned)
|
||||
body, err := io.ReadAll(got.Body)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, got.Body.Close())
|
||||
assert.Equal(t, "important data", string(body), "versioned=%v", versioned)
|
||||
|
||||
deleteBucket(t, client, bucketName)
|
||||
}
|
||||
}
|
||||
|
||||
func listObjects(t *testing.T, client *s3.Client, bucketName, prefix, delimiter string) *s3.ListObjectsV2Output {
|
||||
|
||||
Reference in New Issue
Block a user