mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-07 00:27:03 +00:00
Chris LuandGitHub
863fec6c3f
* filer: keep the sentinel when CreateEntry reports an update failure CreateEntry flattened the error UpdateEntry wraps, so errors.Is stopped matching and ErrExistingIsDirectory and ErrExistingIsFile never reached the S3 mapper, which answered a retryable 500 instead. * s3: let a key that is a prefix of other keys be an object S3 keys are flat, so "a/b" and "a/b/c" are independent objects that coexist in either write order. The filer stores a key as a path, so one of them has to live on the directory the other is nested under. Writing the nested key first refused the prefix key outright. Writing it second promoted the file to a directory, which kept its data but lost the key: an empty object left nothing to recognise it by and disappeared, and one with data listed under a trailing slash it never had. Mark the directory that carries such a key, and write the object onto it when the path is already a directory. The mark makes an empty prefix object visible to listings and readable by GET and HEAD, keeps the empty folder cleaner off it, and lists it under the key it was written with. Deleting the key strips the mark back off along with the data. * filer: keep a TTL off a directory that stands for an object An expired entry is deleted a row at a time, so expiring a directory removes it and leaves everything under it unreachable. Promoting a file to a directory carried its TTL across, and a promoted file is exactly the one that has keys nested under it. Drop the TTL on promotion, and leave one an older build wrote alone. The lifecycle worker still expires the object, through the delete that leaves the directory behind. * s3: delete the null version of a key other keys are nested under The routed delete cannot remove an entry that other keys live under, and answered a retryable 500 rather than falling back to the lock path the unversioned delete already falls back to. That path then looked the entry up under the bucket with the whole key as its name, so the demote wrote it back one directory too high and failed as not found. Fall back on any non-precondition error, and split the key before deleting it. Trailing-slash directory markers with children reach the same delete. * filer: keep the sentinel when MkFile and Mkdir report a create failure Same flattening one layer out: every mkFile caller lost the sentinel, so a CopyObject onto a key that other keys are nested under answered a retryable 500 where a PutObject of the same key answers 409. * s3: copy and rename a key that other keys are nested under Such a key is stored on the directory those keys live in, and copy and rename both refused it: the source lookup maps every directory entry to NoSuchKey, so a key a plain GET serves could not be copied or moved, and the destination side refused it as a directory conflict. The source is read through a view of the entry as the object it names. The destination is written the way a PutObject of that key writes it. A rename at either end copies the object's own data across and strips it off the source key rather than going through AtomicRenameEntry, which moves a directory by moving everything under it - the nested keys are not part of what is being renamed.
547 lines
21 KiB
Go
547 lines
21 KiB
Go
package example
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
"net/http"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/aws/awserr"
|
|
v1credentials "github.com/aws/aws-sdk-go/aws/credentials"
|
|
v1signer "github.com/aws/aws-sdk-go/aws/signer/v4"
|
|
"github.com/aws/aws-sdk-go/service/s3"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// TestS3PrefixObjectKeys covers keys that are a strict prefix of other keys: S3's
|
|
// namespace is flat, so "collision/foo" and "collision/foo/bar" are independent
|
|
// objects that coexist in either write order.
|
|
func TestS3PrefixObjectKeys(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test in short mode")
|
|
}
|
|
|
|
cluster, err := startMiniCluster(t)
|
|
require.NoError(t, err)
|
|
defer cluster.Stop()
|
|
|
|
put := func(t *testing.T, bucket, key string, body []byte) {
|
|
t.Helper()
|
|
_, err := cluster.s3Client.PutObject(&s3.PutObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(key),
|
|
Body: bytes.NewReader(body),
|
|
})
|
|
require.NoError(t, err, "put %s", key)
|
|
}
|
|
// read checks both paths a client reaches an object by, since a directory entry
|
|
// carrying an object is served by neither the directory nor the plain object path
|
|
// alone.
|
|
read := func(t *testing.T, bucket, key string, want []byte) {
|
|
t.Helper()
|
|
head, err := cluster.s3Client.HeadObject(&s3.HeadObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
require.NoError(t, err, "head %s", key)
|
|
assert.Equal(t, int64(len(want)), aws.Int64Value(head.ContentLength), "head %s", key)
|
|
|
|
get, err := cluster.s3Client.GetObject(&s3.GetObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
require.NoError(t, err, "get %s", key)
|
|
defer get.Body.Close()
|
|
got, err := io.ReadAll(get.Body)
|
|
require.NoError(t, err, "read %s", key)
|
|
assert.Equal(t, want, got, "get %s", key)
|
|
}
|
|
// gone checks the key answers as absent rather than lingering on a directory
|
|
// entry that outlived the object.
|
|
gone := func(t *testing.T, bucket, key string) {
|
|
t.Helper()
|
|
_, err := cluster.s3Client.GetObject(&s3.GetObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
var missing awserr.RequestFailure
|
|
require.ErrorAs(t, err, &missing, "get %s", key)
|
|
assert.Equal(t, http.StatusNotFound, missing.StatusCode(), "get %s", key)
|
|
|
|
_, err = cluster.s3Client.HeadObject(&s3.HeadObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
require.ErrorAs(t, err, &missing, "head %s", key)
|
|
assert.Equal(t, http.StatusNotFound, missing.StatusCode(), "head %s", key)
|
|
}
|
|
listKeys := func(t *testing.T, bucket string) []string {
|
|
t.Helper()
|
|
resp, err := cluster.s3Client.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: aws.String(bucket)})
|
|
require.NoError(t, err)
|
|
keys := collectKeys(resp.Contents)
|
|
sort.Strings(keys)
|
|
return keys
|
|
}
|
|
|
|
body := []byte("prefix object")
|
|
// Distinct bodies, so a read that resolves to the wrong entry cannot pass.
|
|
nested := []byte("nested under the prefix object")
|
|
|
|
// The reported order: the nested key is written first, so the prefix key has to
|
|
// land on a path the filer already holds a directory at.
|
|
t.Run("ChildFirst", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-child-first-")
|
|
put(t, bucket, "collision/foo/bar", nested)
|
|
put(t, bucket, "collision/foo", body)
|
|
|
|
assert.Equal(t, []string{"collision/foo", "collision/foo/bar"}, listKeys(t, bucket))
|
|
read(t, bucket, "collision/foo", body)
|
|
read(t, bucket, "collision/foo/bar", nested)
|
|
})
|
|
|
|
// The opposite order used to keep the prefix key's data but hide the key.
|
|
t.Run("PrefixFirst", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-first-")
|
|
put(t, bucket, "collision/foo", body)
|
|
put(t, bucket, "collision/foo/bar", nested)
|
|
|
|
assert.Equal(t, []string{"collision/foo", "collision/foo/bar"}, listKeys(t, bucket))
|
|
read(t, bucket, "collision/foo", body)
|
|
read(t, bucket, "collision/foo/bar", nested)
|
|
})
|
|
|
|
// An empty object leaves no chunks, content or mime behind, so it is the case a
|
|
// promoted directory carries no other trace of.
|
|
t.Run("EmptyObject", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-empty-")
|
|
put(t, bucket, "a/foo/bar", nil)
|
|
put(t, bucket, "a/foo", nil)
|
|
put(t, bucket, "b/foo", nil)
|
|
put(t, bucket, "b/foo/bar", nil)
|
|
|
|
assert.Equal(t, []string{"a/foo", "a/foo/bar", "b/foo", "b/foo/bar"}, listKeys(t, bucket))
|
|
read(t, bucket, "a/foo", []byte{})
|
|
read(t, bucket, "b/foo", []byte{})
|
|
})
|
|
|
|
// The key has no trailing slash, and the keys nested under it still roll up into
|
|
// their own CommonPrefix.
|
|
t.Run("Delimiter", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-delimiter-")
|
|
put(t, bucket, "collision/foo/bar", nested)
|
|
put(t, bucket, "collision/foo", body)
|
|
put(t, bucket, "collision/other", body)
|
|
|
|
resp, err := cluster.s3Client.ListObjectsV2(&s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucket),
|
|
Prefix: aws.String("collision/"),
|
|
Delimiter: aws.String("/"),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
keys := collectKeys(resp.Contents)
|
|
sort.Strings(keys)
|
|
assert.Equal(t, []string{"collision/foo", "collision/other"}, keys)
|
|
assert.Equal(t, []string{"collision/foo/"}, collectPrefixes(resp.CommonPrefixes))
|
|
|
|
// Listing the prefix itself names only what is under it.
|
|
resp, err = cluster.s3Client.ListObjectsV2(&s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucket),
|
|
Prefix: aws.String("collision/foo/"),
|
|
Delimiter: aws.String("/"),
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"collision/foo/bar"}, collectKeys(resp.Contents))
|
|
assert.Empty(t, collectPrefixes(resp.CommonPrefixes))
|
|
})
|
|
|
|
// The key and the CommonPrefix its nested keys fold into come off one filer
|
|
// entry, so a page boundary must not drop either of them.
|
|
t.Run("Paged", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-paged-")
|
|
for _, key := range []string{"foo", "foo/bar", "foobar", "other", "zed", "zed/a"} {
|
|
put(t, bucket, key, body)
|
|
}
|
|
|
|
for _, maxKeys := range []int64{1, 2, 3, 4, 5} {
|
|
var keys, prefixes []string
|
|
var token *string
|
|
for page := 0; page < 12; page++ {
|
|
resp, err := cluster.s3Client.ListObjectsV2(&s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucket),
|
|
Delimiter: aws.String("/"),
|
|
MaxKeys: aws.Int64(maxKeys),
|
|
ContinuationToken: token,
|
|
})
|
|
require.NoError(t, err)
|
|
// One entry over the budget is the documented same-entry exception;
|
|
// anything more means the unsigned budget wrapped.
|
|
assert.LessOrEqual(t, int64(len(resp.Contents)+len(resp.CommonPrefixes)), maxKeys+1,
|
|
"maxKeys=%d page %d", maxKeys, page)
|
|
keys = append(keys, collectKeys(resp.Contents)...)
|
|
prefixes = append(prefixes, collectPrefixes(resp.CommonPrefixes)...)
|
|
if !aws.BoolValue(resp.IsTruncated) {
|
|
token = nil
|
|
break
|
|
}
|
|
token = resp.NextContinuationToken
|
|
require.NotNil(t, token, "a truncated page must name where to resume")
|
|
}
|
|
require.Nil(t, token, "maxKeys=%d did not finish", maxKeys)
|
|
assert.Equal(t, []string{"foo", "foobar", "other", "zed"}, keys, "maxKeys=%d", maxKeys)
|
|
assert.Equal(t, []string{"foo/", "zed/"}, prefixes, "maxKeys=%d", maxKeys)
|
|
}
|
|
})
|
|
|
|
// Versioning reaches a prefix object from two directions: a suspended bucket
|
|
// writes the null version at the key's own path, and a bucket versioned later
|
|
// finds one already sitting there. Both leave a key that is a directory with
|
|
// version history beside it.
|
|
t.Run("Versioned", func(t *testing.T) {
|
|
setVersioning := func(t *testing.T, bucket, status string) {
|
|
t.Helper()
|
|
_, err := cluster.s3Client.PutBucketVersioning(&s3.PutBucketVersioningInput{
|
|
Bucket: aws.String(bucket),
|
|
VersioningConfiguration: &s3.VersioningConfiguration{Status: aws.String(status)},
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
// Both write orders, in a bucket that is versioned and in one where versioning
|
|
// was suspended - the suspended one is the case that writes at the key's path.
|
|
for _, state := range []string{"Enabled", "Suspended"} {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-"+strings.ToLower(state)+"-")
|
|
setVersioning(t, bucket, "Enabled")
|
|
if state == "Suspended" {
|
|
setVersioning(t, bucket, "Suspended")
|
|
}
|
|
put(t, bucket, "child/foo/bar", nested)
|
|
put(t, bucket, "child/foo", body)
|
|
put(t, bucket, "prefix/foo", body)
|
|
put(t, bucket, "prefix/foo/bar", nested)
|
|
|
|
assert.Equal(t, []string{"child/foo", "child/foo/bar", "prefix/foo", "prefix/foo/bar"},
|
|
listKeys(t, bucket), state)
|
|
for _, key := range []string{"child/foo", "prefix/foo"} {
|
|
read(t, bucket, key, body)
|
|
}
|
|
for _, key := range []string{"child/foo/bar", "prefix/foo/bar"} {
|
|
read(t, bucket, key, nested)
|
|
}
|
|
}
|
|
|
|
// A prefix object written before versioning is the key's null version. Removing
|
|
// that version by id must not take the keys nested under it with it.
|
|
bucket := createTestBucket(t, cluster, "test-prefix-nullversion-")
|
|
put(t, bucket, "collision/foo/bar", nested)
|
|
put(t, bucket, "collision/foo", body)
|
|
setVersioning(t, bucket, "Enabled")
|
|
newer := []byte("written after versioning was enabled")
|
|
versioned, err := cluster.s3Client.PutObject(&s3.PutObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String("collision/foo"),
|
|
Body: bytes.NewReader(newer),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
for _, v := range []struct {
|
|
id string
|
|
want []byte
|
|
}{{"null", body}, {aws.StringValue(versioned.VersionId), newer}} {
|
|
got, err := cluster.s3Client.GetObject(&s3.GetObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String("collision/foo"),
|
|
VersionId: aws.String(v.id),
|
|
})
|
|
require.NoError(t, err, "get version %s", v.id)
|
|
body, err := io.ReadAll(got.Body)
|
|
require.NoError(t, err)
|
|
got.Body.Close()
|
|
assert.Equal(t, v.want, body, "get version %s", v.id)
|
|
}
|
|
|
|
_, err = cluster.s3Client.DeleteObject(&s3.DeleteObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String("collision/foo"),
|
|
VersionId: aws.String("null"),
|
|
})
|
|
require.NoError(t, err, "the null version sits on a directory other keys live in")
|
|
|
|
read(t, bucket, "collision/foo", newer)
|
|
read(t, bucket, "collision/foo/bar", nested)
|
|
remaining, err := cluster.s3Client.ListObjectVersions(&s3.ListObjectVersionsInput{
|
|
Bucket: aws.String(bucket),
|
|
Prefix: aws.String("collision/foo"),
|
|
})
|
|
require.NoError(t, err)
|
|
for _, v := range remaining.Versions {
|
|
if aws.StringValue(v.Key) != "collision/foo" {
|
|
// collision/foo/bar predates versioning too, and keeps its null version.
|
|
continue
|
|
}
|
|
assert.NotEqual(t, "null", aws.StringValue(v.VersionId), "the null version was deleted")
|
|
}
|
|
})
|
|
|
|
// The two listings walk the tree differently, and a prefix object is the entry
|
|
// they disagree about: it is a directory the version listing descends through and
|
|
// a key at the same time. They have to name the same keys and the same prefixes.
|
|
t.Run("VersionListingMatchesObjectListing", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-parity-")
|
|
for _, key := range []string{"foo", "foo/bar", "other", "a/foo", "a/foo/bar", "a/z"} {
|
|
put(t, bucket, key, body)
|
|
}
|
|
|
|
for _, q := range []struct{ prefix, delimiter string }{
|
|
{"", ""},
|
|
{"foo/", ""},
|
|
{"a/", ""},
|
|
{"a/foo/", ""},
|
|
{"", "/"},
|
|
{"a/", "/"},
|
|
} {
|
|
name := "prefix=" + q.prefix + " delimiter=" + q.delimiter
|
|
|
|
objects, err := cluster.s3Client.ListObjectsV2(&s3.ListObjectsV2Input{
|
|
Bucket: aws.String(bucket),
|
|
Prefix: aws.String(q.prefix),
|
|
Delimiter: aws.String(q.delimiter),
|
|
})
|
|
require.NoError(t, err, name)
|
|
|
|
versions, err := cluster.s3Client.ListObjectVersions(&s3.ListObjectVersionsInput{
|
|
Bucket: aws.String(bucket),
|
|
Prefix: aws.String(q.prefix),
|
|
Delimiter: aws.String(q.delimiter),
|
|
})
|
|
require.NoError(t, err, name)
|
|
|
|
versionKeys := make([]string, 0, len(versions.Versions))
|
|
for _, v := range versions.Versions {
|
|
versionKeys = append(versionKeys, aws.StringValue(v.Key))
|
|
}
|
|
versionPrefixes := make([]string, 0, len(versions.CommonPrefixes))
|
|
for _, p := range versions.CommonPrefixes {
|
|
versionPrefixes = append(versionPrefixes, aws.StringValue(p.Prefix))
|
|
}
|
|
sort.Strings(versionKeys)
|
|
sort.Strings(versionPrefixes)
|
|
|
|
objectKeys := collectKeys(objects.Contents)
|
|
objectPrefixes := collectPrefixes(objects.CommonPrefixes)
|
|
sort.Strings(objectKeys)
|
|
sort.Strings(objectPrefixes)
|
|
|
|
assert.Equal(t, objectKeys, versionKeys, "keys, %s", name)
|
|
assert.Equal(t, objectPrefixes, versionPrefixes, "prefixes, %s", name)
|
|
}
|
|
})
|
|
|
|
// A prefix object written before versioning is the key's null version, so the
|
|
// version written after it has to take the latest flag off it.
|
|
t.Run("VersionedAfterPrefixObject", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-versioned-")
|
|
put(t, bucket, "collision/foo/bar", nested)
|
|
put(t, bucket, "collision/foo", body)
|
|
_, err := cluster.s3Client.PutBucketVersioning(&s3.PutBucketVersioningInput{
|
|
Bucket: aws.String(bucket),
|
|
VersioningConfiguration: &s3.VersioningConfiguration{Status: aws.String("Enabled")},
|
|
})
|
|
require.NoError(t, err)
|
|
newer := []byte("written after versioning was enabled")
|
|
put(t, bucket, "collision/foo", newer)
|
|
|
|
resp, err := cluster.s3Client.ListObjectVersions(&s3.ListObjectVersionsInput{Bucket: aws.String(bucket)})
|
|
require.NoError(t, err)
|
|
|
|
latest := map[string]int{}
|
|
var nullSize int64 = -1
|
|
for _, v := range resp.Versions {
|
|
if aws.BoolValue(v.IsLatest) {
|
|
latest[aws.StringValue(v.Key)]++
|
|
}
|
|
if aws.StringValue(v.Key) == "collision/foo" && aws.StringValue(v.VersionId) == "null" {
|
|
nullSize = aws.Int64Value(v.Size)
|
|
assert.False(t, aws.BoolValue(v.IsLatest), "the newer version is the latest one")
|
|
}
|
|
}
|
|
assert.Equal(t, 1, latest["collision/foo"], "exactly one version of a key is the latest")
|
|
assert.Equal(t, int64(len(body)), nullSize, "the null version keeps the prefix object's size")
|
|
read(t, bucket, "collision/foo", newer)
|
|
read(t, bucket, "collision/foo/bar", nested)
|
|
})
|
|
|
|
// A key that other keys are nested under is a copy source and a copy destination
|
|
// like any other. The keys nested under either end are not part of the copy.
|
|
t.Run("Copy", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-copy-")
|
|
copyObject := func(t *testing.T, src, dst string) {
|
|
t.Helper()
|
|
_, err := cluster.s3Client.CopyObject(&s3.CopyObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(dst),
|
|
CopySource: aws.String(bucket + "/" + src),
|
|
})
|
|
require.NoError(t, err, "copy %s to %s", src, dst)
|
|
}
|
|
|
|
put(t, bucket, "collision/foo", body)
|
|
put(t, bucket, "collision/foo/bar", nested)
|
|
|
|
// Out of a prefix object, into a key of its own.
|
|
copyObject(t, "collision/foo", "plain")
|
|
read(t, bucket, "plain", body)
|
|
read(t, bucket, "collision/foo", body)
|
|
read(t, bucket, "collision/foo/bar", nested)
|
|
|
|
// Into a key that other keys are nested under.
|
|
put(t, bucket, "target/child", nested)
|
|
copyObject(t, "plain", "target")
|
|
read(t, bucket, "target", body)
|
|
read(t, bucket, "target/child", nested)
|
|
|
|
// And between two of them.
|
|
put(t, bucket, "other", []byte("copied between prefix keys"))
|
|
copyObject(t, "other", "collision/foo")
|
|
read(t, bucket, "collision/foo", []byte("copied between prefix keys"))
|
|
read(t, bucket, "collision/foo/bar", nested)
|
|
|
|
assert.Equal(t, []string{"collision/foo", "collision/foo/bar", "other", "plain", "target", "target/child"},
|
|
listKeys(t, bucket))
|
|
})
|
|
|
|
// Rename moves the object off the key without moving the keys nested under it,
|
|
// which is not what the filer's atomic rename of a directory would do.
|
|
t.Run("Rename", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-rename-")
|
|
renameObject := func(t *testing.T, src, dst string) {
|
|
t.Helper()
|
|
req, _ := http.NewRequest(http.MethodPut, cluster.s3Endpoint+"/"+bucket+"/"+dst+"?renameObject=", nil)
|
|
req.Header.Set("x-amz-rename-source", "/"+bucket+"/"+src)
|
|
signer := v1signer.NewSigner(v1credentials.NewStaticCredentials(testAccessKey, testSecretKey, ""))
|
|
_, err := signer.Sign(req, nil, "s3", testRegion, time.Now())
|
|
require.NoError(t, err)
|
|
|
|
resp, err := (&http.Client{Timeout: 20 * time.Second}).Do(req)
|
|
require.NoError(t, err, "rename %s to %s", src, dst)
|
|
defer resp.Body.Close()
|
|
io.Copy(io.Discard, resp.Body)
|
|
require.Equal(t, http.StatusOK, resp.StatusCode, "rename %s to %s", src, dst)
|
|
}
|
|
|
|
put(t, bucket, "collision/foo", body)
|
|
put(t, bucket, "collision/foo/bar", nested)
|
|
|
|
// Off a prefix object: the key goes, the keys under it stay.
|
|
renameObject(t, "collision/foo", "moved")
|
|
read(t, bucket, "moved", body)
|
|
read(t, bucket, "collision/foo/bar", nested)
|
|
gone(t, bucket, "collision/foo")
|
|
|
|
// Onto a key other keys are nested under.
|
|
put(t, bucket, "target/child", nested)
|
|
renameObject(t, "moved", "target")
|
|
read(t, bucket, "target", body)
|
|
read(t, bucket, "target/child", nested)
|
|
gone(t, bucket, "moved")
|
|
|
|
assert.Equal(t, []string{"collision/foo/bar", "target", "target/child"}, listKeys(t, bucket))
|
|
})
|
|
|
|
// A directory SeaweedFS keeps its own state in is not a prefix a key can be
|
|
// stored on: the object would replace that state with its own.
|
|
t.Run("ReservedDirectory", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-reserved-")
|
|
_, err := cluster.s3Client.PutBucketVersioning(&s3.PutBucketVersioningInput{
|
|
Bucket: aws.String(bucket),
|
|
VersioningConfiguration: &s3.VersioningConfiguration{Status: aws.String("Enabled")},
|
|
})
|
|
require.NoError(t, err)
|
|
put(t, bucket, "foo", body)
|
|
_, err = cluster.s3Client.PutBucketVersioning(&s3.PutBucketVersioningInput{
|
|
Bucket: aws.String(bucket),
|
|
VersioningConfiguration: &s3.VersioningConfiguration{Status: aws.String("Suspended")},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
_, err = cluster.s3Client.PutObject(&s3.PutObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String("foo.versions"),
|
|
Body: bytes.NewReader(body),
|
|
})
|
|
// Not just any error: a permanently impossible write must not come back as a
|
|
// 500 the SDK retries.
|
|
var refused awserr.RequestFailure
|
|
require.ErrorAs(t, err, &refused, "the version history of foo is not a prefix of foo.versions")
|
|
assert.Equal(t, http.StatusConflict, refused.StatusCode())
|
|
assert.Equal(t, "ExistingObjectIsDirectory", refused.Code())
|
|
|
|
versions, err := cluster.s3Client.ListObjectVersions(&s3.ListObjectVersionsInput{Bucket: aws.String(bucket)})
|
|
require.NoError(t, err)
|
|
require.Len(t, versions.Versions, 1)
|
|
assert.Equal(t, "foo", aws.StringValue(versions.Versions[0].Key))
|
|
read(t, bucket, "foo", body)
|
|
|
|
// The multipart staging folder is the other one, and an in-flight upload has
|
|
// to survive the attempt.
|
|
staging := createTestBucket(t, cluster, "test-prefix-uploads-")
|
|
created, err := cluster.s3Client.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
|
|
Bucket: aws.String(staging),
|
|
Key: aws.String("mp.bin"),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
_, err = cluster.s3Client.PutObject(&s3.PutObjectInput{
|
|
Bucket: aws.String(staging),
|
|
Key: aws.String(".uploads"),
|
|
Body: bytes.NewReader(body),
|
|
})
|
|
require.ErrorAs(t, err, &refused, "the multipart staging folder is not a prefix of .uploads")
|
|
assert.Equal(t, http.StatusConflict, refused.StatusCode())
|
|
assert.Equal(t, "ExistingObjectIsDirectory", refused.Code())
|
|
|
|
uploads, err := cluster.s3Client.ListMultipartUploads(&s3.ListMultipartUploadsInput{Bucket: aws.String(staging)})
|
|
require.NoError(t, err)
|
|
require.Len(t, uploads.Uploads, 1)
|
|
assert.Equal(t, "mp.bin", aws.StringValue(uploads.Uploads[0].Key))
|
|
_, err = cluster.s3Client.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
|
|
Bucket: aws.String(staging),
|
|
Key: aws.String("mp.bin"),
|
|
UploadId: created.UploadId,
|
|
})
|
|
require.NoError(t, err)
|
|
})
|
|
|
|
// Either key can be deleted without touching the other.
|
|
t.Run("Delete", func(t *testing.T) {
|
|
bucket := createTestBucket(t, cluster, "test-prefix-delete-")
|
|
put(t, bucket, "collision/foo/bar", nested)
|
|
put(t, bucket, "collision/foo", body)
|
|
|
|
_, err := cluster.s3Client.DeleteObject(&s3.DeleteObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String("collision/foo"),
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"collision/foo/bar"}, listKeys(t, bucket))
|
|
read(t, bucket, "collision/foo/bar", nested)
|
|
gone(t, bucket, "collision/foo")
|
|
|
|
put(t, bucket, "collision/foo", body)
|
|
_, err = cluster.s3Client.DeleteObject(&s3.DeleteObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String("collision/foo/bar"),
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, []string{"collision/foo"}, listKeys(t, bucket))
|
|
read(t, bucket, "collision/foo", body)
|
|
gone(t, bucket, "collision/foo/bar")
|
|
})
|
|
}
|