diff --git a/backend/posix/dir_windows.go b/backend/posix/dir_windows.go index a9fe4cad..31eb6da4 100644 --- a/backend/posix/dir_windows.go +++ b/backend/posix/dir_windows.go @@ -57,7 +57,8 @@ const errDirectory = syscall.Errno(267) // errInvalidName is Windows ERROR_INVALID_NAME (123): "The filename, directory // name, or volume label syntax is incorrect." Windows returns this when a path // component exceeds the filesystem name-length limit — the POSIX equivalent is -// ENAMETOOLONG. +// ENAMETOOLONG. It also returns it for a name with an invalid character, which +// a key isn't stored under const errInvalidName = syscall.Errno(123) // errDirNotEmpty is Windows ERROR_DIR_NOT_EMPTY (145): "The directory is not diff --git a/backend/posix/keyname.go b/backend/posix/keyname.go new file mode 100644 index 00000000..e289cdc6 --- /dev/null +++ b/backend/posix/keyname.go @@ -0,0 +1,227 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package posix + +import ( + "io/fs" + "os" + "slices" + "strings" + "unicode/utf8" +) + +// An object is stored at the path of its key relative to the bucket +// directory, one path element per "/" separated key segment. Windows can't +// name a file after every key segment: the characters " * : < > ? \ | and +// the control characters U+0001-U+001F are invalid, a trailing dot or space +// is stripped, and a device name such as CON or NUL opens the device. With +// encodeKeys, the characters that make a segment such a name are stored as +// the private use character U+F000 plus the character code, the convention +// Cygwin and WSL use for the invalid characters: every invalid character, +// the trailing dot or space, and the first letter of a device name. A private +// use character U+F000-U+F07F of the key is stored after U+F000, so that a +// stored name is the name of one key only. +const ( + // nameEscape is stored before a key character in the range of the + // characters that stand for other characters. + nameEscape = '\uf000' + // nameLast is the last character that stands for another character. + nameLast = '\uf07f' +) + +// encodeKey returns the path, relative to the bucket directory, that the +// object key is stored at. +func (p *Posix) encodeKey(key string) string { + if !p.encodeKeys { + return key + } + return encodeWindowsName(key, isReservedName) +} + +// decodeKey returns the object key stored at name, a path relative to the +// bucket directory. +func (p *Posix) decodeKey(name string) string { + if !p.encodeKeys { + return name + } + return decodeWindowsName(name) +} + +// bucketFS returns the file system of the bucket directory with the entries +// named after the keys they store. +func (p *Posix) bucketFS(bucket string) fs.FS { + fsys := os.DirFS(p.BucketPath(bucket)) + if !p.encodeKeys { + return fsys + } + return windowsNameFS{fsys: fsys, isReserved: isReservedName} +} + +// encodeWindowsName returns the name key is stored under on Windows. +// isReserved reports whether a path element is a device name. +func encodeWindowsName(key string, isReserved func(string) bool) string { + segs := strings.Split(key, "/") + changed := false + for i, seg := range segs { + if enc := encodeWindowsSegment(seg, isReserved); enc != seg { + segs[i] = enc + changed = true + } + } + if !changed { + return key + } + return strings.Join(segs, "/") +} + +func encodeWindowsSegment(seg string, isReserved func(string) bool) string { + // these path elements are resolved like on any other platform + if seg == "" || seg == "." || seg == ".." { + return seg + } + + b := make([]byte, 0, len(seg)) + for i := 0; i < len(seg); { + r, size := utf8.DecodeRuneInString(seg[i:]) + switch { + case isInvalidWindowsChar(r): + b = utf8.AppendRune(b, nameEscape+r) + case nameEscape <= r && r <= nameLast: + b = utf8.AppendRune(b, nameEscape) + b = append(b, seg[i:i+size]...) + default: + b = append(b, seg[i:i+size]...) + } + i += size + } + + if c := b[len(b)-1]; c == '.' || c == ' ' { + b = utf8.AppendRune(b[:len(b)-1], nameEscape+rune(c)) + } + + name := string(b) + // device names start with an ASCII letter + if name[0] < utf8.RuneSelf && isReserved(name) { + name = string(nameEscape+rune(name[0])) + name[1:] + } + return name +} + +func isInvalidWindowsChar(r rune) bool { + switch r { + case '"', '*', ':', '<', '>', '?', '\\', '|': + return true + } + return 0 < r && r < ' ' +} + +// isDOSDeviceName reports whether the path element name, which has no path +// separator or colon, is a device name on Windows 10: a device name followed +// by nothing, by spaces or by an extension. Windows 11 no longer treats a +// device name with an extension as a device. +func isDOSDeviceName(name string) bool { + base, _, _ := strings.Cut(name, ".") + base = strings.TrimRight(base, " ") + switch strings.ToUpper(base) { + case "CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$": + return true + } + if len(base) < 4 { + return false + } + switch strings.ToUpper(base[:3]) { + case "COM", "LPT": + switch base[3:] { + case "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u00b9", "\u00b2", "\u00b3": + return true + } + } + return false +} + +// decodeWindowsName returns the key stored under name on Windows. +func decodeWindowsName(name string) string { + var b []byte + for i := 0; i < len(name); { + r, size := utf8.DecodeRuneInString(name[i:]) + if r < nameEscape || r > nameLast { + if b != nil { + b = append(b, name[i:i+size]...) + } + i += size + continue + } + if b == nil { + b = append(make([]byte, 0, len(name)), name[:i]...) + } + if r == nameEscape && i+size < len(name) { + _, next := utf8.DecodeRuneInString(name[i+size:]) + b = append(b, name[i+size:i+size+next]...) + i += size + next + continue + } + if r == nameEscape { + b = append(b, name[i:i+size]...) + } else { + b = append(b, byte(r-nameEscape)) + } + i += size + } + if b == nil { + return name + } + return string(b) +} + +// windowsNameFS presents the files of fsys, a bucket directory, under the +// keys they're stored at by encodeWindowsName, in the byte order of the keys. +// Entries whose names no key is stored under are left out: the files a key +// can't be read from aren't listed. +type windowsNameFS struct { + fsys fs.FS + isReserved func(string) bool +} + +func (w windowsNameFS) Open(name string) (fs.File, error) { + return w.fsys.Open(encodeWindowsName(name, w.isReserved)) +} + +func (w windowsNameFS) Stat(name string) (fs.FileInfo, error) { + return fs.Stat(w.fsys, encodeWindowsName(name, w.isReserved)) +} + +func (w windowsNameFS) ReadDir(name string) ([]fs.DirEntry, error) { + ents, err := fs.ReadDir(w.fsys, encodeWindowsName(name, w.isReserved)) + keyEnts := make([]fs.DirEntry, 0, len(ents)) + for _, ent := range ents { + key := decodeWindowsName(ent.Name()) + if encodeWindowsName(key, w.isReserved) != ent.Name() { + continue + } + keyEnts = append(keyEnts, keyDirEntry{DirEntry: ent, key: key}) + } + slices.SortFunc(keyEnts, func(a, b fs.DirEntry) int { + return strings.Compare(a.Name(), b.Name()) + }) + return keyEnts, err +} + +// keyDirEntry is a directory entry named after the key segment it stores. +type keyDirEntry struct { + fs.DirEntry + key string +} + +func (e keyDirEntry) Name() string { return e.key } diff --git a/backend/posix/keyname_other.go b/backend/posix/keyname_other.go new file mode 100644 index 00000000..9a1bacbd --- /dev/null +++ b/backend/posix/keyname_other.go @@ -0,0 +1,26 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !windows + +package posix + +// encodeKeysDefault stores the keys under their own names. +const encodeKeysDefault = false + +// isReservedName reports whether the path element name is a device name on +// Windows 10. Keys are only encoded for Windows on another platform in tests. +func isReservedName(name string) bool { + return isDOSDeviceName(name) +} diff --git a/backend/posix/keyname_test.go b/backend/posix/keyname_test.go new file mode 100644 index 00000000..70e8283e --- /dev/null +++ b/backend/posix/keyname_test.go @@ -0,0 +1,450 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package posix + +import ( + "context" + "io" + "io/fs" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "testing/fstest" + "unicode/utf8" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/versity/versitygw/s3response" +) + +func TestEncodeWindowsName(t *testing.T) { + for _, test := range []struct{ key, name string }{ + {"normal.txt", "normal.txt"}, + {"plus+test %#.txt", "plus+test %#.txt"}, + {"question?.txt", "question\uf03f.txt"}, + {"asterisk*.txt", "asterisk\uf02a.txt"}, + {"中文/空 格+%?#.txt", "中文/空 格+%\uf03f#.txt"}, + {`<>:"|?*`, "\uf03c\uf03e\uf03a\uf022\uf07c\uf03f\uf02a"}, + {`back\slash`, "back\uf05cslash"}, + {`..\..\escape`, "..\uf05c..\uf05cescape"}, + {"C:drive", "C\uf03adrive"}, + {"ctrl\x01\x1f\tchar", "ctrl\uf001\uf01f\uf009char"}, + {"del\x7f", "del\x7f"}, + {"trailingdot.", "trailingdot\uf02e"}, + {"trailingspace ", "trailingspace\uf020"}, + {"dots...", "dots..\uf02e"}, + {"...", "..\uf02e"}, + {" ", "\uf020"}, + {" leadingspace", " leadingspace"}, + {"dotdir./spacedir /file", "dotdir\uf02e/spacedir\uf020/file"}, + {"dir/", "dir/"}, + {"/a//b/", "/a//b/"}, + {"./a/../b", "./a/../b"}, + {"CON", "\uf043ON"}, + {"con", "\uf063on"}, + {"nul.txt", "\uf06eul.txt"}, + {"Aux.tar.gz", "\uf041ux.tar.gz"}, + {"prn .txt", "\uf070rn .txt"}, + {"aux/x", "\uf061ux/x"}, + {"x/COM1", "x/\uf043OM1"}, + {"LPT9.log", "\uf04cPT9.log"}, + {"lpt\u00b9", "\uf06cpt\u00b9"}, + {"CONIN$", "\uf043ONIN$"}, + {"conout$.x", "\uf063onout$.x"}, + {"con ", "con\uf020"}, + {"con.", "con\uf02e"}, + {"com0", "com0"}, + {"CONSOLE", "CONSOLE"}, + {"nul?", "nul\uf03f"}, + {"pua\uf03f.txt", "pua\uf000\uf03f.txt"}, + {"\uf000", "\uf000\uf000"}, + {"\uf07f\uf080", "\uf000\uf07f\uf080"}, + {"fullwidth\uff1f", "fullwidth\uff1f"}, + {"emoji😀", "emoji😀"}, + } { + name := encodeWindowsName(test.key, isDOSDeviceName) + if name != test.name { + t.Errorf("encode %+q: got %+q, want %+q", test.key, name, test.name) + } + if key := decodeWindowsName(name); key != test.key { + t.Errorf("decode %+q: got %+q, want %+q", name, key, test.key) + } + checkWindowsName(t, name) + } +} + +func TestDecodeWindowsName(t *testing.T) { + for _, test := range []struct{ name, key string }{ + // names Cygwin and WSL store invalid characters under + {"a\uf03f\uf02a\uf03a\uf05c", `a?*:\`}, + {"\uf001\uf01f", "\x01\x1f"}, + // names no key is stored under + {"a\uf000", "a\uf000"}, + {"a\uf041", "aA"}, + {"a\uf02fb", "a/b"}, + {"a\uf080", "a\uf080"}, + {"bad\xffutf8", "bad\xffutf8"}, + } { + if key := decodeWindowsName(test.name); key != test.key { + t.Errorf("decode %+q: got %+q, want %+q", test.name, key, test.key) + } + } +} + +func FuzzWindowsName(f *testing.F) { + for _, key := range []string{"a?b", "dir./CON", "x\uf000\uf03f", "nul .txt ", "a//b/./c/../"} { + f.Add(key) + } + f.Fuzz(func(t *testing.T, key string) { + if !utf8.ValidString(key) || strings.ContainsRune(key, 0) { + t.Skip("not an S3 key") + } + name := encodeWindowsName(key, isDOSDeviceName) + if got := decodeWindowsName(name); got != key { + t.Fatalf("decode(encode(%+q)) = %+q", key, got) + } + checkWindowsName(t, name) + }) +} + +// TestWindowsNameFS checks that the bucket directory is walked over the keys, +// in the byte order of the keys, and that the entries no key is stored at +// are left out. +func TestWindowsNameFS(t *testing.T) { + enc := func(key string) string { return encodeWindowsName(key, isDOSDeviceName) } + fsys := fstest.MapFS{} + for _, key := range []string{"a", "a?", "aA", "a\uf03f", "a.", "a!/x", "a*/y", "CON/z"} { + fsys[enc(key)] = &fstest.MapFile{} + } + fsys["a\uf041"] = &fstest.MapFile{} // is "aA" once decoded + fsys["CON2\uf000"] = &fstest.MapFile{} // is "CON2\uf000\uf000" once encoded + w := windowsNameFS{fsys: fsys, isReserved: isDOSDeviceName} + + ents, err := fs.ReadDir(w, ".") + if err != nil { + t.Fatal(err) + } + var names []string + for _, ent := range ents { + names = append(names, ent.Name()) + } + want := []string{"CON", "a", "a!", "a*", "a.", "a?", "aA", "a\uf03f"} + if !slices.Equal(names, want) { + t.Errorf("read dir: got %+q, want %+q", names, want) + } + + ents, err = fs.ReadDir(w, "CON") + if err != nil || len(ents) != 1 || ents[0].Name() != "z" { + t.Errorf("read dir CON: got %v, %v", ents, err) + } + if _, err := fs.Stat(w, "a*/y"); err != nil { + t.Errorf("stat a*/y: %v", err) + } + if _, err := fs.ReadFile(w, "a?"); err != nil { + t.Errorf("read a?: %v", err) + } +} + +// TestEncodedKeys checks that with encodeKeys every operation finds an object +// at the encoded key, that the keys are returned as they were written, and +// that no name Windows rejects or rewrites is stored. The listings are the +// ones of AWS S3 for the same keys. +func TestEncodedKeys(t *testing.T) { + for mode, mkMeta := range metaModes(t) { + if mode == "xattr" && runtime.GOOS == "windows" { + continue + } + t.Run(mode, func(t *testing.T) { + root, versionDir := t.TempDir(), t.TempDir() + storer, opts := mkMeta(t) + opts.AbsolutePaths = true + opts.VersioningDir = versionDir + opts.CopyObjectThreshold = 5 * 1024 * 1024 * 1024 + p, err := New(root, storer, opts) + if err != nil { + t.Fatalf("new posix: %v", err) + } + defer p.Shutdown() + p.encodeKeys = true + + ctx := context.Background() + bucket := "bucket" + createTestBucket(t, p, bucket) + + keys := []string{ + "question?.txt", "中文/空 格+%?#.txt", `<>:"|`, `back\slash`, `..\..\escape`, + "C:drive.txt", "ctrl\x01char", "trailingdot.", "trailingspace ", + "dotdir./spacedir /file", "CON", "nul.txt", "aux/x", "pua\uf03f.txt", + "sort/a", "sort/a.", "sort/a?", "sort/a?b", "sort/aA", "sort/a\uf03f", + "sort/a*/y", "sort/a:z", "sort/a!", "dirobj?/", + } + // a directory object holds no data + body := func(key string) string { + if strings.HasSuffix(key, "/") { + return "" + } + return key + } + for _, key := range keys { + if err := putTestObject(t, p, bucket, key, body(key)); err != nil { + t.Fatalf("put %+q: %v", key, err) + } + } + for _, key := range keys { + // the frontend always passes a range + out, err := p.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &key, Range: aws.String("")}) + if err != nil { + t.Fatalf("get %+q: %v", key, err) + } + var data []byte + if out.Body != nil { + data, _ = io.ReadAll(out.Body) + out.Body.Close() + } + if string(data) != body(key) { + t.Errorf("get %+q: got %+q", key, data) + } + } + + sorted := slices.Clone(keys) + slices.Sort(sorted) + list := func(prefix, delim, startAfter string) ([]string, []string) { + t.Helper() + var contents, prefixes []string + // the frontend always passes a continuation token + token := aws.String("") + for { + out, err := p.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, Prefix: &prefix, Delimiter: &delim, + StartAfter: &startAfter, ContinuationToken: token, MaxKeys: aws.Int32(2), + }) + if err != nil { + t.Fatalf("list: %v", err) + } + for _, obj := range out.Contents { + contents = append(contents, *obj.Key) + } + for _, cp := range out.CommonPrefixes { + prefixes = append(prefixes, *cp.Prefix) + } + if !*out.IsTruncated { + return contents, prefixes + } + token = out.NextContinuationToken + } + } + checkList := func(got, want []string) { + t.Helper() + if !slices.Equal(got, want) { + t.Errorf("list: got %+q, want %+q", got, want) + } + } + contents, _ := list("", "", "") + checkList(contents, sorted) + contents, _ = list("sort/a?", "", "") + checkList(contents, []string{"sort/a?", "sort/a?b"}) + contents, prefixes := list("sort/", "/", "") + checkList(contents, []string{"sort/a", "sort/a!", "sort/a.", "sort/a:z", "sort/a?", "sort/a?b", "sort/aA", "sort/a\uf03f"}) + checkList(prefixes, []string{"sort/a*/"}) + contents, prefixes = list("sort/", "?", "") + checkList(contents, []string{"sort/a", "sort/a!", "sort/a*/y", "sort/a.", "sort/a:z", "sort/aA", "sort/a\uf03f"}) + checkList(prefixes, []string{"sort/a?"}) + contents, _ = list("sort/", "", "sort/a?") + checkList(contents, []string{"sort/a?b", "sort/aA", "sort/a\uf03f"}) + v1, err := p.ListObjects(ctx, &s3.ListObjectsInput{ + Bucket: &bucket, Prefix: aws.String("sort/"), Marker: aws.String("sort/a."), MaxKeys: aws.Int32(1000), + }) + if err != nil { + t.Fatalf("list v1: %v", err) + } + contents = nil + for _, obj := range v1.Contents { + contents = append(contents, *obj.Key) + } + checkList(contents, []string{"sort/a:z", "sort/a?", "sort/a?b", "sort/aA", "sort/a\uf03f"}) + + if err := p.PutObjectTagging(ctx, bucket, "question?.txt", "", map[string]string{"k": "v"}); err != nil { + t.Fatalf("put tagging: %v", err) + } + tags, err := p.GetObjectTagging(ctx, bucket, "question?.txt", "") + if err != nil || tags["k"] != "v" { + t.Errorf("get tagging: got %v, %v", tags, err) + } + + copyKey := "copy*dst?." + _, err = p.CopyObject(ctx, s3response.CopyObjectInput{ + Bucket: &bucket, Key: ©Key, CopySource: aws.String(bucket + "/" + "question?.txt"), + ExpectedBucketOwner: aws.String(""), MetadataDirective: types.MetadataDirectiveCopy, + }) + if err != nil { + t.Fatalf("copy: %v", err) + } + if data, _ := getTestObject(t, p, bucket, copyKey); string(data) != "question?.txt" { + t.Errorf("get copy: got %+q", data) + } + + mpKey := `mp?*:<>|"\.` + mp, err := p.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{Bucket: &bucket, Key: &mpKey}) + if err != nil { + t.Fatalf("create mp: %v", err) + } + uploads, err := p.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{ + Bucket: &bucket, Prefix: aws.String("mp?"), MaxUploads: aws.Int32(1000), + }) + if err != nil || len(uploads.Uploads) != 1 || uploads.Uploads[0].Key != mpKey { + t.Errorf("list mp uploads: got %+v, %v", uploads.Uploads, err) + } + part, err := p.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: &bucket, Key: &mpKey, UploadId: &mp.UploadId, PartNumber: aws.Int32(1), + Body: strings.NewReader("part"), ContentLength: aws.Int64(4), + }) + if err != nil { + t.Fatalf("upload part: %v", err) + } + done, _, err := p.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, Key: &mpKey, UploadId: &mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: []types.CompletedPart{{PartNumber: aws.Int32(1), ETag: part.ETag}}, + }, + }) + if err != nil || *done.Key != mpKey { + t.Fatalf("complete mp: got %v, %v", done.Key, err) + } + if data, _ := getTestObject(t, p, bucket, mpKey); string(data) != "part" { + t.Errorf("get mp object: got %+q", data) + } + + copyPartKey := "mp2*" + mp2, err := p.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{Bucket: &bucket, Key: ©PartKey}) + if err != nil { + t.Fatalf("create mp: %v", err) + } + _, err = p.UploadPartCopy(ctx, &s3.UploadPartCopyInput{ + Bucket: &bucket, Key: ©PartKey, UploadId: &mp2.UploadId, PartNumber: aws.Int32(1), + CopySource: aws.String(bucket + "/" + "question?.txt"), CopySourceRange: aws.String(""), + }) + if err != nil { + t.Fatalf("upload part copy: %v", err) + } + err = p.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{Bucket: &bucket, Key: ©PartKey, UploadId: &mp2.UploadId}) + if err != nil { + t.Fatalf("abort mp: %v", err) + } + + if got := p.NormalizeObjectKey(bucket, `..\..\escape`); got != `..\..\escape` { + t.Errorf("normalize: got %+q", got) + } + + if err := p.PutBucketVersioning(ctx, bucket, types.BucketVersioningStatusEnabled); err != nil { + t.Fatalf("put bucket versioning: %v", err) + } + if err := p.PutObjectLockConfiguration(ctx, bucket, []byte(`{"Enabled":true}`)); err != nil { + t.Fatalf("put object lock configuration: %v", err) + } + lockKey := "nul.txt" + if err := p.PutObjectLegalHold(ctx, bucket, lockKey, "", true); err != nil { + t.Fatalf("put legal hold: %v", err) + } + head, err := p.HeadObject(ctx, &s3.HeadObjectInput{Bucket: &bucket, Key: &lockKey}) + if err != nil || head.ObjectLockLegalHoldStatus != types.ObjectLockLegalHoldStatusOn { + t.Errorf("head legal hold: got %+v, %v", head, err) + } + retention := []byte(`{"Mode":"GOVERNANCE"}`) + if err := p.PutObjectRetention(ctx, bucket, lockKey, "", retention); err != nil { + t.Fatalf("put retention: %v", err) + } + if got, err := p.GetObjectRetention(ctx, bucket, lockKey, ""); err != nil || string(got) != string(retention) { + t.Errorf("get retention: got %s, %v", got, err) + } + + vKey := "trailingdot." + if err := putTestObject(t, p, bucket, vKey, "v2"); err != nil { + t.Fatalf("put version: %v", err) + } + if _, err := p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &vKey}); err != nil { + t.Fatalf("delete: %v", err) + } + versions, err := p.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{ + Bucket: &bucket, Prefix: &vKey, MaxKeys: aws.Int32(1000), + }) + if err != nil { + t.Fatalf("list versions: %v", err) + } + if len(versions.DeleteMarkers) != 1 || *versions.DeleteMarkers[0].Key != vKey || len(versions.Versions) != 2 { + t.Fatalf("list versions: got %+v", versions) + } + for _, v := range versions.Versions { + out, err := p.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &vKey, VersionId: v.VersionId}) + if err != nil { + t.Fatalf("get version: %v", err) + } + data, _ := io.ReadAll(out.Body) + out.Body.Close() + if *v.Key != vKey || (string(data) != vKey && string(data) != "v2") { + t.Errorf("get version %v of %+q: got %+q", *v.VersionId, *v.Key, data) + } + } + _, err = p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &vKey, VersionId: versions.DeleteMarkers[0].VersionId}) + if err != nil { + t.Fatalf("delete marker: %v", err) + } + if data, _ := getTestObject(t, p, bucket, vKey); string(data) != "v2" { + t.Errorf("get restored version: got %+q", data) + } + + for _, dir := range []string{root, versionDir, opts.SideCarDir} { + if dir == "" { + continue + } + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil || path == dir { + return err + } + checkWindowsName(t, d.Name()) + return nil + }) + if err != nil { + t.Fatalf("walk %v: %v", dir, err) + } + } + + for _, key := range append(keys, copyKey, mpKey) { + _, err := p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &key}) + if err != nil { + t.Errorf("delete %+q: %v", key, err) + } + } + }) + } +} + +// checkWindowsName fails the test if the path elements of name can't name a +// file on Windows, or on Windows 10 when not run on Windows. +func checkWindowsName(t *testing.T, name string) { + t.Helper() + for _, elem := range strings.Split(name, "/") { + if elem == "" || elem == "." || elem == ".." { + continue + } + if strings.ContainsFunc(elem, isInvalidWindowsChar) || + strings.HasSuffix(elem, ".") || strings.HasSuffix(elem, " ") || + isReservedName(elem) { + t.Errorf("%+q isn't a valid Windows file name", elem) + } + } +} diff --git a/backend/posix/keyname_windows.go b/backend/posix/keyname_windows.go new file mode 100644 index 00000000..08cf952a --- /dev/null +++ b/backend/posix/keyname_windows.go @@ -0,0 +1,30 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build windows + +package posix + +import "path/filepath" + +// encodeKeysDefault stores the keys under names that are valid on Windows. +const encodeKeysDefault = true + +// isReservedName reports whether Windows opens a device for the path element +// name, which has no path separator or colon. A device name with an extension, +// such as NUL.txt, is a device on Windows 10 but not on Windows 11, which +// filepath.IsLocal asks the system about. +func isReservedName(name string) bool { + return !filepath.IsLocal(name) +} diff --git a/backend/posix/keyname_windows_test.go b/backend/posix/keyname_windows_test.go new file mode 100644 index 00000000..b44e9255 --- /dev/null +++ b/backend/posix/keyname_windows_test.go @@ -0,0 +1,76 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build windows + +package posix + +import ( + "os" + "path/filepath" + "slices" + "testing" +) + +// TestReservedNameWithoutExtension checks that a device name without an +// extension is reserved on every version of Windows. +func TestReservedNameWithoutExtension(t *testing.T) { + for _, name := range []string{"CON", "con", "nul ", "AUX", "PRN", "COM1", "lpt9", "COM\u00b9", "CONIN$", "conout$"} { + if !isReservedName(name) { + t.Errorf("%+q isn't reserved", name) + } + } + for _, name := range []string{"CONSOLE", "COM0", "LPT", "a", "con\uf020"} { + if isReservedName(name) { + t.Errorf("%+q is reserved", name) + } + } +} + +// TestEncodedNamesAreFiles checks that Windows stores a file under the name +// of every key, apart from the keys it would otherwise store it as. +func TestEncodedNamesAreFiles(t *testing.T) { + dir := t.TempDir() + keys := []string{ + "question?", "question", "a:b", "a", `back\slash`, "pipe|<>\"*", "tab\tchar", + "trailingdot.", "trailingdot", "trailingspace ", "trailingspace", + "CON", "nul.txt", "COM1", "LPT\u00b9", "CONIN$", "pua\uf03f", "pua?", + } + for _, key := range keys { + name := filepath.Join(dir, encodeWindowsName(key, isReservedName)) + if err := os.WriteFile(name, []byte(key), 0644); err != nil { + t.Fatalf("write %+q: %v", key, err) + } + } + for _, key := range keys { + data, err := os.ReadFile(filepath.Join(dir, encodeWindowsName(key, isReservedName))) + if err != nil || string(data) != key { + t.Errorf("read %+q: got %+q, %v", key, data, err) + } + } + + ents, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var got []string + for _, ent := range ents { + got = append(got, decodeWindowsName(ent.Name())) + } + slices.Sort(got) + slices.Sort(keys) + if !slices.Equal(got, keys) { + t.Errorf("read dir: got %+q, want %+q", got, keys) + } +} diff --git a/backend/posix/posix.go b/backend/posix/posix.go index fc815cfd..b325f999 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -114,6 +114,10 @@ type Posix struct { // frontend handlers are already validating bucket names validateBucketName bool + // encodeKeys stores the objects under names that are valid on Windows, + // see encodeKey. It's set on Windows. + encodeKeys bool + // defaultEtag is returned for objects that do not have an etag attribute // stored (e.g. files placed on the filesystem outside of versitygw). defaultEtag string @@ -427,6 +431,7 @@ func New(rootdir string, ms meta.MetadataStorer, opts PosixOpts) (*Posix, error) objectLockMode: objectLockMode, enableODirect: opts.EnableODirect, validateBucketName: opts.ValidateBucketNames, + encodeKeys: encodeKeysDefault, actionLimiter: semaphore.NewWeighted(int64(concurrencyOrDefault(opts.Concurrency))), copyObjectThreshold: opts.CopyObjectThreshold, defaultEtag: opts.DefaultEtag, @@ -1596,8 +1601,7 @@ func (p *Posix) ListObjectVersions(ctx context.Context, input *s3.ListObjectVers return s3response.ListVersionsResult{}, err } - fileSystem := os.DirFS(p.BucketPath(bucket)) - results, err := backend.WalkVersions(ctx, fileSystem, prefix, delim, keyMarker, versionIdMarker, max, + results, err := backend.WalkVersions(ctx, p.bucketFS(bucket), prefix, delim, keyMarker, versionIdMarker, max, p.fileToObjVersions(bucket), []string{MetaTmpDir}) if err != nil { return s3response.ListVersionsResult{}, fmt.Errorf("walk %v: %w", bucket, err) @@ -1712,6 +1716,8 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { if d.IsDir() { key = path + "/" } + // the walk is over the keys, the object is stored at the encoded key + path = p.encodeKey(path) // get object info and fill out object data etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey) @@ -1805,7 +1811,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc { } // List all the versions of the object in the versioning directory - versionPath := p.genObjVersionPath(bucket, key) + versionPath := p.genObjVersionPath(bucket, p.encodeKey(key)) dirEnts, err := os.ReadDir(versionPath) if errors.Is(err, fs.ErrNotExist) { return &backend.ObjVersionFuncResult{ @@ -2344,7 +2350,8 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C } bucket := *input.Bucket - object := *input.Key + key := *input.Key + object := p.encodeKey(key) uploadID := *input.UploadId parts := input.MultipartUpload.Parts @@ -2365,7 +2372,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C // the processing slot. A concurrent call with the same uploadId will compute // the same ETag, so it will either find the directory still present (still // processing) or gone (already completed) and react accordingly. - sum := sha256.Sum256([]byte(object)) + sum := sha256.Sum256([]byte(key)) objdirFull := filepath.Join(p.BucketPath(bucket), MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) uploadIDDir := filepath.Join(objdirFull, uploadID) // Compute the default multipart ETag token used for claim naming. @@ -2487,7 +2494,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C return s3response.CompleteMultipartUploadResult{ Bucket: &bucket, ETag: &etag, - Key: &object, + Key: &key, }, "", nil } // Directory is gone: a completed upload is linked before its @@ -2504,7 +2511,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C return res, "", s3err.GetAPIError(s3err.ErrNoSuchUpload) } if isErrNameTooLong(err) { - return res, "", s3err.GetKeyTooLongErr(int64(len(object)), 1024) + return res, "", s3err.GetKeyTooLongErr(int64(len(key)), 1024) } if err != nil { return res, "", fmt.Errorf("stat object: %w", err) @@ -2547,7 +2554,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C return s3response.CompleteMultipartUploadResult{ Bucket: &bucket, ETag: &etag, - Key: &object, + Key: &key, }, string(versionID), nil } if err != nil { @@ -3028,7 +3035,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C return s3response.CompleteMultipartUploadResult{ Bucket: &bucket, ETag: &finalEtag, - Key: &object, + Key: &key, ChecksumCRC32: crc32, ChecksumCRC32C: crc32c, ChecksumSHA1: sha1, @@ -4146,10 +4153,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber)) - srcBucket, srcObject, srcVersionId, err := backend.ParseCopySource(*upi.CopySource) + srcBucket, srcKey, srcVersionId, err := backend.ParseCopySource(*upi.CopySource) if err != nil { return s3response.CopyPartResult{}, err } + srcObject := p.encodeKey(srcKey) if err := p.validateVersionId(srcVersionId); err != nil { return s3response.CopyPartResult{}, err } @@ -4204,12 +4212,12 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) fi, err := os.Stat(objPath) if errors.Is(err, fs.ErrNotExist) { if p.versioningEnabled() && vEnabled { - return s3response.CopyPartResult{}, s3err.GetNoSuchVersionErr(srcObject, srcVersionId) + return s3response.CopyPartResult{}, s3err.GetNoSuchVersionErr(srcKey, srcVersionId) } return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } if isErrNameTooLong(err) { - return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(srcObject)), 1024) + return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(srcKey)), 1024) } if err != nil { return s3response.CopyPartResult{}, fmt.Errorf("stat object: %w", err) @@ -4561,13 +4569,14 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje return s3response.PutObjectOutput{}, err } - name := p.ObjectPath(*po.Bucket, *po.Key) + object := p.encodeKey(*po.Key) + name := p.ObjectPath(*po.Bucket, object) // Fast-fail precondition check before the request body is staged. This // is only advisory: the authoritative check is repeated while holding // the object publish lock at commit time so that evaluation and // publication are atomic under concurrent writers. - err = p.checkPutPreconditions(*po.Bucket, *po.Key, po.IfMatch, po.IfNoneMatch) + err = p.checkPutPreconditions(*po.Bucket, object, po.IfMatch, po.IfNoneMatch) if err != nil { return s3response.PutObjectOutput{}, err } @@ -4616,7 +4625,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } } - if strings.HasSuffix(*po.Key, "/") { + if strings.HasSuffix(object, "/") { // object is directory if contentLength != 0 { // posix directories can't contain data, send error @@ -4627,13 +4636,13 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje // serialize with competing writers to the same key and repeat the // precondition check authoritatively under the publish lock - unlock, err := p.lockObjectPublish(ctx, *po.Bucket, *po.Key) + unlock, err := p.lockObjectPublish(ctx, *po.Bucket, object) if err != nil { return s3response.PutObjectOutput{}, err } defer unlock() - err = p.checkPutPreconditions(*po.Bucket, *po.Key, po.IfMatch, po.IfNoneMatch) + err = p.checkPutPreconditions(*po.Bucket, object, po.IfMatch, po.IfNoneMatch) if err != nil { return s3response.PutObjectOutput{}, err } @@ -4663,7 +4672,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje // In a versioned bucket the directory is kept across versions: its // current version is copied to the versioning directory first. - err = p.snapshotObjVersion(*po.Bucket, *po.Key, vStatus, acct) + err = p.snapshotObjVersion(*po.Bucket, object, vStatus, acct) if err != nil { return s3response.PutObjectOutput{}, err } @@ -4682,7 +4691,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje // The directory outlives the object it replaces, whether or not // the bucket is versioned: the new object starts with none of the // replaced object's attributes. - err = p.clearDirObjectAttrs(*po.Bucket, *po.Key) + err = p.clearDirObjectAttrs(*po.Bucket, object) if err != nil { return s3response.PutObjectOutput{}, err } @@ -4690,7 +4699,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje var versionID string if versioned { if p.isBucketVersioningSuspended(vStatus) { - err = p.deleteNullVersionIdObject(*po.Bucket, *po.Key) + err = p.deleteNullVersionIdObject(*po.Bucket, object) if err != nil { return s3response.PutObjectOutput{}, err } @@ -4699,7 +4708,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } } - err = p.storeObjectMetadata(nil, *po.Bucket, *po.Key, po.Metadata) + err = p.storeObjectMetadata(nil, *po.Bucket, object, po.Metadata) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set object metadata: %w", err) } @@ -4710,21 +4719,21 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } // set etag attribute to signify this dir was specifically put - err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, etagkey, + err = p.meta.StoreAttribute(nil, *po.Bucket, object, etagkey, []byte(dirETag)) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) } // set "application/x-directory" content-type - err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, contentTypeHdr, + err = p.meta.StoreAttribute(nil, *po.Bucket, object, contentTypeHdr, []byte(backend.DirContentType)) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err) } if getString(po.WebsiteRedirectLocation) != "" { - err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, websiteRedirectHdr, + err = p.meta.StoreAttribute(nil, *po.Bucket, object, websiteRedirectHdr, []byte(*po.WebsiteRedirectLocation)) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set website-redirect-location attr: %w", err) @@ -4739,13 +4748,13 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje checksum.SetSum(checksumAlgorithm, &expectedSum) - err = p.storeChecksums(nil, *po.Bucket, *po.Key, checksum) + err = p.storeChecksums(nil, *po.Bucket, object, checksum) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) } if versionID != "" { - err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, versionIdKey, []byte(versionID)) + err = p.meta.StoreAttribute(nil, *po.Bucket, object, versionIdKey, []byte(versionID)) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err) } @@ -4819,7 +4828,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } f, err := p.openTmpFile(p.ObjectPath(*po.Bucket, MetaTmpDir), - *po.Bucket, *po.Key, contentLength, acct, doFalloc, p.forceNoTmpFile, odirectAllowed) + *po.Bucket, object, contentLength, acct, doFalloc, p.forceNoTmpFile, odirectAllowed) if err != nil { if errors.Is(err, syscall.EDQUOT) { drainBody(po.Body) @@ -4917,7 +4926,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje // The body is fully staged; serialize the commit phase (condition // evaluation, metadata stores, and final link) with competing writers // to the same key. - unlock, err := p.lockObjectPublish(ctx, *po.Bucket, *po.Key) + unlock, err := p.lockObjectPublish(ctx, *po.Bucket, object) if err != nil { return s3response.PutObjectOutput{}, err } @@ -4925,7 +4934,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje if conditional { // authoritative precondition check under the publish lock - err = p.checkPutPreconditions(*po.Bucket, *po.Key, po.IfMatch, po.IfNoneMatch) + err = p.checkPutPreconditions(*po.Bucket, object, po.IfMatch, po.IfNoneMatch) if err != nil { return s3response.PutObjectOutput{}, err } @@ -4941,7 +4950,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje // Snapshot the object that is about to be replaced only after the // conditional check has succeeded and while holding the publish lock. - verr := p.snapshotObjVersion(*po.Bucket, *po.Key, vStatus, acct) + verr := p.snapshotObjVersion(*po.Bucket, object, vStatus, acct) if verr != nil { return s3response.PutObjectOutput{}, verr } @@ -4953,7 +4962,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje if versionID != "" { keep = append(keep, versionIdKey) } - err = p.meta.ReplaceObject(*po.Bucket, *po.Key, keep) + err = p.meta.ReplaceObject(*po.Bucket, object, keep) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("remove replaced object attributes: %w", err) } @@ -4962,7 +4971,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje // null versionId object from versioning directory // if it exists and the versioning status is Suspended if p.isBucketVersioningSuspended(vStatus) { - err = p.deleteNullVersionIdObject(*po.Bucket, *po.Key) + err = p.deleteNullVersionIdObject(*po.Bucket, object) if err != nil { return s3response.PutObjectOutput{}, err } @@ -4995,17 +5004,17 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } checksum.SetSum(checksumAlgorithm, &sum) - err = p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum) + err = p.storeChecksums(f.File(), *po.Bucket, object, checksum) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err) } - err = p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, etagkey, []byte(etag)) + err = p.meta.StoreAttribute(f.File(), *po.Bucket, object, etagkey, []byte(etag)) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err) } - err = p.storeObjectMetaProperties(f.File(), *po.Bucket, *po.Key, + err = p.storeObjectMetaProperties(f.File(), *po.Bucket, object, metaProperties{ ContentType: po.ContentType, ContentEncoding: po.ContentEncoding, @@ -5021,7 +5030,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje } if versionID != "" && versionID != nullVersionId { - err := p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, versionIdKey, []byte(versionID)) + err := p.meta.StoreAttribute(f.File(), *po.Bucket, object, versionIdKey, []byte(versionID)) if err != nil { return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err) } @@ -5142,7 +5151,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( } bucket := *input.Bucket - object := *input.Key + object := p.encodeKey(*input.Key) isDir := strings.HasSuffix(object, "/") if !p.isBucketValid(bucket) { @@ -5198,7 +5207,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return &s3.DeleteObjectOutput{}, nil } if isErrNameTooLong(err) { - return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024) } if err != nil { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) @@ -5284,7 +5293,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil } if isErrNameTooLong(err) { - return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024) } if err != nil { return nil, fmt.Errorf("stat object: %w", err) @@ -5452,7 +5461,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( err = os.Remove(filepath.Join(versionPath, *input.VersionId)) if isErrNameTooLong(err) { - return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024) } if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, *input.VersionId) @@ -5480,7 +5489,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( fi, err := os.Stat(objpath) if isErrNameTooLong(err) { - return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) + return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024) } if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { // AWS returns success if the object does not exist @@ -5840,7 +5849,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge return nil, err } - object := *input.Key + object := p.encodeKey(*input.Key) if versionId != "" { vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { @@ -6179,7 +6188,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. } bucket := *input.Bucket - object := *input.Key + object := p.encodeKey(*input.Key) if !p.isBucketValid(bucket) { return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) @@ -6349,7 +6358,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. } var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus - status, err := p.GetObjectLegalHold(withCtxNoSlot(ctx), bucket, object, versionId) + status, err := p.GetObjectLegalHold(withCtxNoSlot(ctx), bucket, *input.Key, versionId) if err == nil { if *status { objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn @@ -6360,7 +6369,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. var objectLockMode types.ObjectLockMode var objectLockRetainUntilDate *time.Time - retention, err := p.GetObjectRetention(withCtxNoSlot(ctx), bucket, object, versionId) + retention, err := p.GetObjectRetention(withCtxNoSlot(ctx), bucket, *input.Key, versionId) if err == nil { var config types.ObjectLockRetention if err := json.Unmarshal(retention, &config); err == nil { @@ -6506,7 +6515,8 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput return s3response.CopyObjectOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, srcBucket) } dstBucket := *input.Bucket - dstObject := *input.Key + dstKey := *input.Key + dstObject := p.encodeKey(dstKey) if !p.isBucketValid(dstBucket) { return s3response.CopyObjectOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, dstBucket) @@ -6552,6 +6562,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput vEnabled := p.isBucketVersioningEnabled(vStatus) origSrcObject := srcObject + srcObject = p.encodeKey(srcObject) if srcVersionId != "" { if !p.versioningEnabled() || !vEnabled { return s3response.CopyObjectOutput{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, srcVersionId) @@ -6792,7 +6803,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput return s3response.CopyObjectOutput{}, err } - err = p.PutObjectTagging(withCtxNoSlot(ctx), dstBucket, dstObject, "", tags) + err = p.PutObjectTagging(withCtxNoSlot(ctx), dstBucket, dstKey, "", tags) if err != nil { return s3response.CopyObjectOutput{}, err } @@ -6802,7 +6813,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput // settings of a newly written one err = p.putObjectLockSettings(ctx, s3response.PutObjectInput{ Bucket: &dstBucket, - Key: &dstObject, + Key: &dstKey, ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate, ObjectLockMode: input.ObjectLockMode, ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus, @@ -6834,7 +6845,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput putObjectInput := s3response.PutObjectInput{ Bucket: &dstBucket, - Key: &dstObject, + Key: &dstKey, Body: srcBody, ContentLength: &contentLength, ChecksumAlgorithm: checksums.Algorithm, @@ -6975,8 +6986,7 @@ func (p *Posix) ListObjectsParametrized(ctx context.Context, input *s3.ListObjec return s3response.ListObjectsResult{}, err } - fileSystem := os.DirFS(p.BucketPath(bucket)) - results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, + results, err := backend.Walk(ctx, p.bucketFS(bucket), prefix, delim, marker, maxkeys, customFileToObj(bucket, true), []string{MetaTmpDir}) if err != nil { return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err) @@ -7002,7 +7012,10 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { } } - return func(path string, d fs.DirEntry) (s3response.Object, error) { + return func(key string, d fs.DirEntry) (s3response.Object, error) { + // the walk is over the keys, the object is stored at the encoded key + path := p.encodeKey(key) + var owner *types.Owner // Retrieve the object owner data from bucket ACL, if fetchOwner is true // All the objects in the bucket are owned by the bucket owner @@ -7052,7 +7065,7 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { return s3response.Object{ ETag: &etag, - Key: &path, + Key: &key, LastModified: &mtime, Size: &size, StorageClass: types.ObjectStorageClassStandard, @@ -7101,7 +7114,7 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { return s3response.Object{ ETag: &etag, - Key: &path, + Key: &key, LastModified: &mtime, Size: &size, StorageClass: types.ObjectStorageClassStandard, @@ -7158,8 +7171,7 @@ func (p *Posix) ListObjectsV2Parametrized(ctx context.Context, input *s3.ListObj return s3response.ListObjectsV2Result{}, err } - fileSystem := os.DirFS(p.BucketPath(bucket)) - results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, + results, err := backend.Walk(ctx, p.bucketFS(bucket), prefix, delim, marker, maxkeys, customFileToObj(bucket, fetchOwner), []string{MetaTmpDir}) if err != nil { return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err) @@ -7325,6 +7337,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } + object = p.encodeKey(object) bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) if err != nil { return nil, err @@ -7392,6 +7405,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } + object = p.encodeKey(object) bucket, object, err = p.objVersionAttrPath(bucket, object, versionId) if err != nil { return err @@ -7826,6 +7840,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } + object = p.encodeKey(object) err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return err @@ -7886,6 +7901,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } + object = p.encodeKey(object) err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return nil, err @@ -7946,6 +7962,7 @@ func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } + object = p.encodeKey(object) err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return err @@ -7993,6 +8010,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI if !p.isBucketValid(bucket) { return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket) } + object = p.encodeKey(object) err = p.doesBucketAndObjectExist(bucket, object, versionId) if err != nil { return nil, err @@ -8133,18 +8151,19 @@ func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response. // attempt that escapes bucket), but the result names an S3 key, not a host // path: on Windows filepath.Join/Rel would return it with "\" separators, // which callers building a policy-resource ARN or match string must never -// see, so it's converted back to "/" before returning. +// see, so it's converted back to "/" before returning. The key is resolved +// as the path it's stored at, in which a "\" of the key is no separator. func (p *Posix) NormalizeObjectKey(bucket, object string) string { - fullPath := filepath.Join(bucket, object) + fullPath := filepath.Join(bucket, p.encodeKey(object)) key, err := filepath.Rel(filepath.Clean(bucket), fullPath) if err != nil { - return filepath.ToSlash(fullPath) + return p.decodeKey(filepath.ToSlash(fullPath)) } if key == "." { return "" } - return filepath.ToSlash(key) + return p.decodeKey(filepath.ToSlash(key)) } func (p *Posix) storeChecksums(f *os.File, bucket, object string, chs s3response.Checksum) error { diff --git a/tests/integration/PutObject.go b/tests/integration/PutObject.go index 7f14b83b..21e038c5 100644 --- a/tests/integration/PutObject.go +++ b/tests/integration/PutObject.go @@ -53,21 +53,10 @@ func PutObject_special_chars(s *S3Conf) error { "my?key", "my^key", "my{}key", "my%key", "my`key", "my[]key", "my~key", "my<>key", "my|key", "my#key", } - if !s.azureTests && !s.windowsTests { - // azure and windows cannot handle backslashes in object names: - // on Windows, '\' is a path separator so 'my\key' is stored as 'my/key' + if !s.azureTests { + // azure cannot handle backslashes in object names objnames = append(objnames, "my\\key") } - if s.windowsTests { - // ':', '?', '<', '>', '|', '*' are not valid filename characters on Windows - var filtered []string - for _, name := range objnames { - if !strings.ContainsAny(name, `:?<>|*`) { - filtered = append(filtered, name) - } - } - objnames = filtered - } return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { objs, err := putObjects(s3client, objnames, bucket) @@ -1731,21 +1720,13 @@ func PutObject_false_negative_object_names(s *S3Conf) error { "..%2f..%2f..%2fsecret/file.txt", // URL-encoded "..%c0%af..%c0%afetc/passwd", // UTF-8 overlong trick "dir/%c0%ae%c0%ae/%c0%ae%c0%ae/etc/passwd", // overlong UTF-8 encoding + ".../.../.../target.txt", + "dir/%20../file.txt", // encoded space + "..\\u2215..\\u2215etc/passwd", // Unicode division slash + "logs/latest -> /etc/passwd", // symlink attacks //TODO: add this test case in advanced routing // "/etc/passwd" // absolute path injection } - if !s.windowsTests { - // Windows strips trailing dots from path components, making '...' an - // invalid directory name (the filesystem rejects MkdirAll with it). - objs = append(objs, ".../.../.../target.txt") - objs = append(objs, "dir/%20../file.txt") // encoded space - // literal backslashes are treated as path separators on Windows - objs = append(objs, "..\\u2215..\\u2215etc/passwd") // Unicode division slash - // On Windows, '>' is an invalid filename character. The key - // 'logs/latest -> /etc/passwd' creates a directory component - // 'latest -> ' (containing '>') which MkdirAll rejects. - objs = append(objs, "logs/latest -> /etc/passwd") // symlink attacks - } sort.Strings(objs) diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index c0f82614..209ac3ce 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -1073,6 +1073,7 @@ func TestPosix(ts *TestState) { ts.Run(PutObject_overwrite_file_obj_with_nested_obj) ts.Run(PutObject_dir_obj_with_data) ts.Run(PutObject_with_slashes) + ts.Run(PutObject_windows_reserved_names) ts.Run(CreateMultipartUpload_dir_obj) ts.Run(PutObject_name_too_long) ts.Run(HeadObject_name_too_long) @@ -1185,6 +1186,7 @@ func TestScoutfs(ts *TestState) { ts.Run(PutObject_overwrite_file_obj_with_nested_obj) ts.Run(PutObject_dir_obj_with_data) ts.Run(PutObject_with_slashes) + ts.Run(PutObject_windows_reserved_names) ts.Run(CreateMultipartUpload_dir_obj) ts.Run(PutObject_name_too_long) ts.Run(HeadObject_name_too_long) @@ -2084,9 +2086,7 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_CopyObject_to_itself) ts.Run(Versioning_CopyObject_to_itself_from_the_current_version) ts.Run(Versioning_CopyObject_conditional_writes) - if !ts.conf.windowsTests { - ts.Run(Versioning_CopyObject_special_chars) - } + ts.Run(Versioning_CopyObject_special_chars) // HeadObject action ts.Run(Versioning_HeadObject_invalid_versionId) if !ts.conf.windowsTests { @@ -3645,6 +3645,7 @@ func GetIntTests() IntTests { "PutObject_overwrite_file_obj_with_nested_obj": PutObject_overwrite_file_obj_with_nested_obj, "PutObject_dir_obj_with_data": PutObject_dir_obj_with_data, "PutObject_with_slashes": PutObject_with_slashes, + "PutObject_windows_reserved_names": PutObject_windows_reserved_names, "PutObject_race_with_delete": PutObject_race_with_delete, "CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj, "PutObject_sse_c_not_implemented": PutObject_sse_c_not_implemented, diff --git a/tests/integration/posix.go b/tests/integration/posix.go index facc0e8b..f3eb9e97 100644 --- a/tests/integration/posix.go +++ b/tests/integration/posix.go @@ -17,6 +17,10 @@ package integration import ( "context" "fmt" + "io" + "slices" + "sort" + "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -114,6 +118,102 @@ func PutObject_with_slashes(s *S3Conf) error { }) } +func PutObject_windows_reserved_names(s *S3Conf) error { + testName := "PutObject_windows_reserved_names" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + // Windows can't name a file after these keys: each one is next to + // the key Windows would store it as + keys := []string{ + "question?", "question", "asterisk*", `quote"`, "colon:name", "C:drive", + "lt", "pipe|", "tab\tchar", + `back\slash`, "back/slash", `..\..\escape`, + "trailingdot.", "trailingdot", "trailingspace ", "trailingspace", + "dotdir./file", "dotdir/file", "spacedir /file", "spacedir/file", + "CON", "nul.txt", "aux/obj", "com1", "LPT9.log", "CONIN$", + "pua\uf03f", "pua?", + } + for _, key := range keys { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err := s3client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &bucket, + Key: &key, + Body: strings.NewReader(key), + }) + cancel() + if err != nil { + return fmt.Errorf("put %q: %w", key, err) + } + } + + for _, key := range keys { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + out, err := s3client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + }) + if err != nil { + cancel() + return fmt.Errorf("get %q: %w", key, err) + } + body, err := io.ReadAll(out.Body) + out.Body.Close() + cancel() + if err != nil { + return fmt.Errorf("read %q: %w", key, err) + } + if string(body) != key { + return fmt.Errorf("expected the body of %q to be the key, instead got %q", key, body) + } + } + + sort.Strings(keys) + var wantContents []string + var wantPrefixes []string + for _, key := range keys { + prefix, _, found := strings.Cut(key, "/") + if !found { + wantContents = append(wantContents, key) + } else if !slices.Contains(wantPrefixes, prefix+"/") { + wantPrefixes = append(wantPrefixes, prefix+"/") + } + } + + for _, delimiter := range []string{"", "/"} { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: &bucket, + Delimiter: &delimiter, + }) + cancel() + if err != nil { + return err + } + var contents, prefixes []string + for _, obj := range res.Contents { + contents = append(contents, *obj.Key) + } + for _, cp := range res.CommonPrefixes { + prefixes = append(prefixes, *cp.Prefix) + } + + want, wantCps := keys, []string(nil) + if delimiter != "" { + want, wantCps = wantContents, wantPrefixes + } + if !slices.Equal(contents, want) { + return fmt.Errorf("expected the objects listed with delimiter %q to be %q, instead got %q", + delimiter, want, contents) + } + if !slices.Equal(prefixes, wantCps) { + return fmt.Errorf("expected the common prefixes listed with delimiter %q to be %q, instead got %q", + delimiter, wantCps, prefixes) + } + } + + return nil + }) +} + func CreateMultipartUpload_dir_obj(s *S3Conf) error { testName := "CreateMultipartUpload_dir_obj" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { diff --git a/tests/integration/presigned_urls.go b/tests/integration/presigned_urls.go index c4c29003..a23969cf 100644 --- a/tests/integration/presigned_urls.go +++ b/tests/integration/presigned_urls.go @@ -857,10 +857,6 @@ func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error { testName := "PresignedAuth_Put_GetObject_with_UTF8_chars" return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { obj := "my-$%^&*;" - if s.windowsTests { - // '*' is not a valid filename character on Windows - obj = "my-$%^&;" - } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj})