mirror of
https://github.com/versity/versitygw.git
synced 2026-10-22 06:12:39 +00:00
fix: store posix object keys with Windows-invalid characters under encoded names on Windows
Fixes #2513 On Windows the posix backend stored a key as `bucket/key` verbatim, but Windows rejects `" * : < > ? \ |` and control characters in file names, so such a PUT failed with `ERROR_INVALID_NAME`, which `isErrNameTooLong` reported as a misleading `KeyTooLongError`. The same mapping did worse elsewhere: `\` acted as a path separator, so `a\b` aliased `a/b` and `..\..\x` resolved outside the bucket because `IsObjectNameValid` only splits on `/`; `a:b` wrote an NTFS alternate data stream; Win32 strips a trailing dot or space, so `foo.` overwrote `foo`; and device names such as `CON` or `nul.txt` opened the device and silently discarded the data. With `encodeKeys`, enabled by default only on Windows, `encodeKey` stores each key segment under a name Windows accepts: every invalid character, a trailing dot or space, and the first letter of a device name are stored as the private use character `U+F000` plus the character code, the convention Cygwin and WSL use for invalid characters. A key character in `U+F000`-`U+F07F` is stored after `U+F000`, so every stored name belongs to exactly one key. Device names are checked with `filepath.IsLocal`, which asks the system, so `CON.txt` stays a plain file on Windows 11 where it is valid. Keys that need no encoding keep their names, so existing data is unaffected, and nothing changes on other platforms. Public backend methods keep taking raw keys and encode them once for every path, metadata, version-hash and publish-lock use, while responses, errors and the multipart `objname` attribute keep the raw key and multipart upload directories keep hashing it. Listings walk `bucketFS`, which decodes the entry names, sorts them in key byte order to match S3 and skips names no key is stored under. `HeadObject` passes the raw key to `GetObjectLegalHold` and `GetObjectRetention` to avoid encoding twice, and `NormalizeObjectKey` resolves the encoded key, so policy resources see `a\b` as one literal key.
This commit is contained in:
@@ -57,7 +57,8 @@ const errDirectory = syscall.Errno(267)
|
||||
// errInvalidName is Windows ERROR_INVALID_NAME (123): "The filename, directory
|
||||
// name, or volume label syntax is incorrect." Windows returns this when a path
|
||||
// component exceeds the filesystem name-length limit — the POSIX equivalent is
|
||||
// ENAMETOOLONG.
|
||||
// ENAMETOOLONG. It also returns it for a name with an invalid character, which
|
||||
// a key isn't stored under
|
||||
const errInvalidName = syscall.Errno(123)
|
||||
|
||||
// errDirNotEmpty is Windows ERROR_DIR_NOT_EMPTY (145): "The directory is not
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
// Copyright 2026 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package posix
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// An object is stored at the path of its key relative to the bucket
|
||||
// directory, one path element per "/" separated key segment. Windows can't
|
||||
// name a file after every key segment: the characters " * : < > ? \ | and
|
||||
// the control characters U+0001-U+001F are invalid, a trailing dot or space
|
||||
// is stripped, and a device name such as CON or NUL opens the device. With
|
||||
// encodeKeys, the characters that make a segment such a name are stored as
|
||||
// the private use character U+F000 plus the character code, the convention
|
||||
// Cygwin and WSL use for the invalid characters: every invalid character,
|
||||
// the trailing dot or space, and the first letter of a device name. A private
|
||||
// use character U+F000-U+F07F of the key is stored after U+F000, so that a
|
||||
// stored name is the name of one key only.
|
||||
const (
|
||||
// nameEscape is stored before a key character in the range of the
|
||||
// characters that stand for other characters.
|
||||
nameEscape = '\uf000'
|
||||
// nameLast is the last character that stands for another character.
|
||||
nameLast = '\uf07f'
|
||||
)
|
||||
|
||||
// encodeKey returns the path, relative to the bucket directory, that the
|
||||
// object key is stored at.
|
||||
func (p *Posix) encodeKey(key string) string {
|
||||
if !p.encodeKeys {
|
||||
return key
|
||||
}
|
||||
return encodeWindowsName(key, isReservedName)
|
||||
}
|
||||
|
||||
// decodeKey returns the object key stored at name, a path relative to the
|
||||
// bucket directory.
|
||||
func (p *Posix) decodeKey(name string) string {
|
||||
if !p.encodeKeys {
|
||||
return name
|
||||
}
|
||||
return decodeWindowsName(name)
|
||||
}
|
||||
|
||||
// bucketFS returns the file system of the bucket directory with the entries
|
||||
// named after the keys they store.
|
||||
func (p *Posix) bucketFS(bucket string) fs.FS {
|
||||
fsys := os.DirFS(p.BucketPath(bucket))
|
||||
if !p.encodeKeys {
|
||||
return fsys
|
||||
}
|
||||
return windowsNameFS{fsys: fsys, isReserved: isReservedName}
|
||||
}
|
||||
|
||||
// encodeWindowsName returns the name key is stored under on Windows.
|
||||
// isReserved reports whether a path element is a device name.
|
||||
func encodeWindowsName(key string, isReserved func(string) bool) string {
|
||||
segs := strings.Split(key, "/")
|
||||
changed := false
|
||||
for i, seg := range segs {
|
||||
if enc := encodeWindowsSegment(seg, isReserved); enc != seg {
|
||||
segs[i] = enc
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return key
|
||||
}
|
||||
return strings.Join(segs, "/")
|
||||
}
|
||||
|
||||
func encodeWindowsSegment(seg string, isReserved func(string) bool) string {
|
||||
// these path elements are resolved like on any other platform
|
||||
if seg == "" || seg == "." || seg == ".." {
|
||||
return seg
|
||||
}
|
||||
|
||||
b := make([]byte, 0, len(seg))
|
||||
for i := 0; i < len(seg); {
|
||||
r, size := utf8.DecodeRuneInString(seg[i:])
|
||||
switch {
|
||||
case isInvalidWindowsChar(r):
|
||||
b = utf8.AppendRune(b, nameEscape+r)
|
||||
case nameEscape <= r && r <= nameLast:
|
||||
b = utf8.AppendRune(b, nameEscape)
|
||||
b = append(b, seg[i:i+size]...)
|
||||
default:
|
||||
b = append(b, seg[i:i+size]...)
|
||||
}
|
||||
i += size
|
||||
}
|
||||
|
||||
if c := b[len(b)-1]; c == '.' || c == ' ' {
|
||||
b = utf8.AppendRune(b[:len(b)-1], nameEscape+rune(c))
|
||||
}
|
||||
|
||||
name := string(b)
|
||||
// device names start with an ASCII letter
|
||||
if name[0] < utf8.RuneSelf && isReserved(name) {
|
||||
name = string(nameEscape+rune(name[0])) + name[1:]
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func isInvalidWindowsChar(r rune) bool {
|
||||
switch r {
|
||||
case '"', '*', ':', '<', '>', '?', '\\', '|':
|
||||
return true
|
||||
}
|
||||
return 0 < r && r < ' '
|
||||
}
|
||||
|
||||
// isDOSDeviceName reports whether the path element name, which has no path
|
||||
// separator or colon, is a device name on Windows 10: a device name followed
|
||||
// by nothing, by spaces or by an extension. Windows 11 no longer treats a
|
||||
// device name with an extension as a device.
|
||||
func isDOSDeviceName(name string) bool {
|
||||
base, _, _ := strings.Cut(name, ".")
|
||||
base = strings.TrimRight(base, " ")
|
||||
switch strings.ToUpper(base) {
|
||||
case "CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$":
|
||||
return true
|
||||
}
|
||||
if len(base) < 4 {
|
||||
return false
|
||||
}
|
||||
switch strings.ToUpper(base[:3]) {
|
||||
case "COM", "LPT":
|
||||
switch base[3:] {
|
||||
case "1", "2", "3", "4", "5", "6", "7", "8", "9", "\u00b9", "\u00b2", "\u00b3":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// decodeWindowsName returns the key stored under name on Windows.
|
||||
func decodeWindowsName(name string) string {
|
||||
var b []byte
|
||||
for i := 0; i < len(name); {
|
||||
r, size := utf8.DecodeRuneInString(name[i:])
|
||||
if r < nameEscape || r > nameLast {
|
||||
if b != nil {
|
||||
b = append(b, name[i:i+size]...)
|
||||
}
|
||||
i += size
|
||||
continue
|
||||
}
|
||||
if b == nil {
|
||||
b = append(make([]byte, 0, len(name)), name[:i]...)
|
||||
}
|
||||
if r == nameEscape && i+size < len(name) {
|
||||
_, next := utf8.DecodeRuneInString(name[i+size:])
|
||||
b = append(b, name[i+size:i+size+next]...)
|
||||
i += size + next
|
||||
continue
|
||||
}
|
||||
if r == nameEscape {
|
||||
b = append(b, name[i:i+size]...)
|
||||
} else {
|
||||
b = append(b, byte(r-nameEscape))
|
||||
}
|
||||
i += size
|
||||
}
|
||||
if b == nil {
|
||||
return name
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// windowsNameFS presents the files of fsys, a bucket directory, under the
|
||||
// keys they're stored at by encodeWindowsName, in the byte order of the keys.
|
||||
// Entries whose names no key is stored under are left out: the files a key
|
||||
// can't be read from aren't listed.
|
||||
type windowsNameFS struct {
|
||||
fsys fs.FS
|
||||
isReserved func(string) bool
|
||||
}
|
||||
|
||||
func (w windowsNameFS) Open(name string) (fs.File, error) {
|
||||
return w.fsys.Open(encodeWindowsName(name, w.isReserved))
|
||||
}
|
||||
|
||||
func (w windowsNameFS) Stat(name string) (fs.FileInfo, error) {
|
||||
return fs.Stat(w.fsys, encodeWindowsName(name, w.isReserved))
|
||||
}
|
||||
|
||||
func (w windowsNameFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
||||
ents, err := fs.ReadDir(w.fsys, encodeWindowsName(name, w.isReserved))
|
||||
keyEnts := make([]fs.DirEntry, 0, len(ents))
|
||||
for _, ent := range ents {
|
||||
key := decodeWindowsName(ent.Name())
|
||||
if encodeWindowsName(key, w.isReserved) != ent.Name() {
|
||||
continue
|
||||
}
|
||||
keyEnts = append(keyEnts, keyDirEntry{DirEntry: ent, key: key})
|
||||
}
|
||||
slices.SortFunc(keyEnts, func(a, b fs.DirEntry) int {
|
||||
return strings.Compare(a.Name(), b.Name())
|
||||
})
|
||||
return keyEnts, err
|
||||
}
|
||||
|
||||
// keyDirEntry is a directory entry named after the key segment it stores.
|
||||
type keyDirEntry struct {
|
||||
fs.DirEntry
|
||||
key string
|
||||
}
|
||||
|
||||
func (e keyDirEntry) Name() string { return e.key }
|
||||
@@ -0,0 +1,26 @@
|
||||
// Copyright 2026 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build !windows
|
||||
|
||||
package posix
|
||||
|
||||
// encodeKeysDefault stores the keys under their own names.
|
||||
const encodeKeysDefault = false
|
||||
|
||||
// isReservedName reports whether the path element name is a device name on
|
||||
// Windows 10. Keys are only encoded for Windows on another platform in tests.
|
||||
func isReservedName(name string) bool {
|
||||
return isDOSDeviceName(name)
|
||||
}
|
||||
@@ -0,0 +1,450 @@
|
||||
// Copyright 2026 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
package posix
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/versity/versitygw/s3response"
|
||||
)
|
||||
|
||||
func TestEncodeWindowsName(t *testing.T) {
|
||||
for _, test := range []struct{ key, name string }{
|
||||
{"normal.txt", "normal.txt"},
|
||||
{"plus+test %#.txt", "plus+test %#.txt"},
|
||||
{"question?.txt", "question\uf03f.txt"},
|
||||
{"asterisk*.txt", "asterisk\uf02a.txt"},
|
||||
{"中文/空 格+%?#.txt", "中文/空 格+%\uf03f#.txt"},
|
||||
{`<>:"|?*`, "\uf03c\uf03e\uf03a\uf022\uf07c\uf03f\uf02a"},
|
||||
{`back\slash`, "back\uf05cslash"},
|
||||
{`..\..\escape`, "..\uf05c..\uf05cescape"},
|
||||
{"C:drive", "C\uf03adrive"},
|
||||
{"ctrl\x01\x1f\tchar", "ctrl\uf001\uf01f\uf009char"},
|
||||
{"del\x7f", "del\x7f"},
|
||||
{"trailingdot.", "trailingdot\uf02e"},
|
||||
{"trailingspace ", "trailingspace\uf020"},
|
||||
{"dots...", "dots..\uf02e"},
|
||||
{"...", "..\uf02e"},
|
||||
{" ", "\uf020"},
|
||||
{" leadingspace", " leadingspace"},
|
||||
{"dotdir./spacedir /file", "dotdir\uf02e/spacedir\uf020/file"},
|
||||
{"dir/", "dir/"},
|
||||
{"/a//b/", "/a//b/"},
|
||||
{"./a/../b", "./a/../b"},
|
||||
{"CON", "\uf043ON"},
|
||||
{"con", "\uf063on"},
|
||||
{"nul.txt", "\uf06eul.txt"},
|
||||
{"Aux.tar.gz", "\uf041ux.tar.gz"},
|
||||
{"prn .txt", "\uf070rn .txt"},
|
||||
{"aux/x", "\uf061ux/x"},
|
||||
{"x/COM1", "x/\uf043OM1"},
|
||||
{"LPT9.log", "\uf04cPT9.log"},
|
||||
{"lpt\u00b9", "\uf06cpt\u00b9"},
|
||||
{"CONIN$", "\uf043ONIN$"},
|
||||
{"conout$.x", "\uf063onout$.x"},
|
||||
{"con ", "con\uf020"},
|
||||
{"con.", "con\uf02e"},
|
||||
{"com0", "com0"},
|
||||
{"CONSOLE", "CONSOLE"},
|
||||
{"nul?", "nul\uf03f"},
|
||||
{"pua\uf03f.txt", "pua\uf000\uf03f.txt"},
|
||||
{"\uf000", "\uf000\uf000"},
|
||||
{"\uf07f\uf080", "\uf000\uf07f\uf080"},
|
||||
{"fullwidth\uff1f", "fullwidth\uff1f"},
|
||||
{"emoji😀", "emoji😀"},
|
||||
} {
|
||||
name := encodeWindowsName(test.key, isDOSDeviceName)
|
||||
if name != test.name {
|
||||
t.Errorf("encode %+q: got %+q, want %+q", test.key, name, test.name)
|
||||
}
|
||||
if key := decodeWindowsName(name); key != test.key {
|
||||
t.Errorf("decode %+q: got %+q, want %+q", name, key, test.key)
|
||||
}
|
||||
checkWindowsName(t, name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeWindowsName(t *testing.T) {
|
||||
for _, test := range []struct{ name, key string }{
|
||||
// names Cygwin and WSL store invalid characters under
|
||||
{"a\uf03f\uf02a\uf03a\uf05c", `a?*:\`},
|
||||
{"\uf001\uf01f", "\x01\x1f"},
|
||||
// names no key is stored under
|
||||
{"a\uf000", "a\uf000"},
|
||||
{"a\uf041", "aA"},
|
||||
{"a\uf02fb", "a/b"},
|
||||
{"a\uf080", "a\uf080"},
|
||||
{"bad\xffutf8", "bad\xffutf8"},
|
||||
} {
|
||||
if key := decodeWindowsName(test.name); key != test.key {
|
||||
t.Errorf("decode %+q: got %+q, want %+q", test.name, key, test.key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzWindowsName(f *testing.F) {
|
||||
for _, key := range []string{"a?b", "dir./CON", "x\uf000\uf03f", "nul .txt ", "a//b/./c/../"} {
|
||||
f.Add(key)
|
||||
}
|
||||
f.Fuzz(func(t *testing.T, key string) {
|
||||
if !utf8.ValidString(key) || strings.ContainsRune(key, 0) {
|
||||
t.Skip("not an S3 key")
|
||||
}
|
||||
name := encodeWindowsName(key, isDOSDeviceName)
|
||||
if got := decodeWindowsName(name); got != key {
|
||||
t.Fatalf("decode(encode(%+q)) = %+q", key, got)
|
||||
}
|
||||
checkWindowsName(t, name)
|
||||
})
|
||||
}
|
||||
|
||||
// TestWindowsNameFS checks that the bucket directory is walked over the keys,
|
||||
// in the byte order of the keys, and that the entries no key is stored at
|
||||
// are left out.
|
||||
func TestWindowsNameFS(t *testing.T) {
|
||||
enc := func(key string) string { return encodeWindowsName(key, isDOSDeviceName) }
|
||||
fsys := fstest.MapFS{}
|
||||
for _, key := range []string{"a", "a?", "aA", "a\uf03f", "a.", "a!/x", "a*/y", "CON/z"} {
|
||||
fsys[enc(key)] = &fstest.MapFile{}
|
||||
}
|
||||
fsys["a\uf041"] = &fstest.MapFile{} // is "aA" once decoded
|
||||
fsys["CON2\uf000"] = &fstest.MapFile{} // is "CON2\uf000\uf000" once encoded
|
||||
w := windowsNameFS{fsys: fsys, isReserved: isDOSDeviceName}
|
||||
|
||||
ents, err := fs.ReadDir(w, ".")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for _, ent := range ents {
|
||||
names = append(names, ent.Name())
|
||||
}
|
||||
want := []string{"CON", "a", "a!", "a*", "a.", "a?", "aA", "a\uf03f"}
|
||||
if !slices.Equal(names, want) {
|
||||
t.Errorf("read dir: got %+q, want %+q", names, want)
|
||||
}
|
||||
|
||||
ents, err = fs.ReadDir(w, "CON")
|
||||
if err != nil || len(ents) != 1 || ents[0].Name() != "z" {
|
||||
t.Errorf("read dir CON: got %v, %v", ents, err)
|
||||
}
|
||||
if _, err := fs.Stat(w, "a*/y"); err != nil {
|
||||
t.Errorf("stat a*/y: %v", err)
|
||||
}
|
||||
if _, err := fs.ReadFile(w, "a?"); err != nil {
|
||||
t.Errorf("read a?: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestEncodedKeys checks that with encodeKeys every operation finds an object
|
||||
// at the encoded key, that the keys are returned as they were written, and
|
||||
// that no name Windows rejects or rewrites is stored. The listings are the
|
||||
// ones of AWS S3 for the same keys.
|
||||
func TestEncodedKeys(t *testing.T) {
|
||||
for mode, mkMeta := range metaModes(t) {
|
||||
if mode == "xattr" && runtime.GOOS == "windows" {
|
||||
continue
|
||||
}
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
root, versionDir := t.TempDir(), t.TempDir()
|
||||
storer, opts := mkMeta(t)
|
||||
opts.AbsolutePaths = true
|
||||
opts.VersioningDir = versionDir
|
||||
opts.CopyObjectThreshold = 5 * 1024 * 1024 * 1024
|
||||
p, err := New(root, storer, opts)
|
||||
if err != nil {
|
||||
t.Fatalf("new posix: %v", err)
|
||||
}
|
||||
defer p.Shutdown()
|
||||
p.encodeKeys = true
|
||||
|
||||
ctx := context.Background()
|
||||
bucket := "bucket"
|
||||
createTestBucket(t, p, bucket)
|
||||
|
||||
keys := []string{
|
||||
"question?.txt", "中文/空 格+%?#.txt", `<>:"|`, `back\slash`, `..\..\escape`,
|
||||
"C:drive.txt", "ctrl\x01char", "trailingdot.", "trailingspace ",
|
||||
"dotdir./spacedir /file", "CON", "nul.txt", "aux/x", "pua\uf03f.txt",
|
||||
"sort/a", "sort/a.", "sort/a?", "sort/a?b", "sort/aA", "sort/a\uf03f",
|
||||
"sort/a*/y", "sort/a:z", "sort/a!", "dirobj?/",
|
||||
}
|
||||
// a directory object holds no data
|
||||
body := func(key string) string {
|
||||
if strings.HasSuffix(key, "/") {
|
||||
return ""
|
||||
}
|
||||
return key
|
||||
}
|
||||
for _, key := range keys {
|
||||
if err := putTestObject(t, p, bucket, key, body(key)); err != nil {
|
||||
t.Fatalf("put %+q: %v", key, err)
|
||||
}
|
||||
}
|
||||
for _, key := range keys {
|
||||
// the frontend always passes a range
|
||||
out, err := p.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &key, Range: aws.String("")})
|
||||
if err != nil {
|
||||
t.Fatalf("get %+q: %v", key, err)
|
||||
}
|
||||
var data []byte
|
||||
if out.Body != nil {
|
||||
data, _ = io.ReadAll(out.Body)
|
||||
out.Body.Close()
|
||||
}
|
||||
if string(data) != body(key) {
|
||||
t.Errorf("get %+q: got %+q", key, data)
|
||||
}
|
||||
}
|
||||
|
||||
sorted := slices.Clone(keys)
|
||||
slices.Sort(sorted)
|
||||
list := func(prefix, delim, startAfter string) ([]string, []string) {
|
||||
t.Helper()
|
||||
var contents, prefixes []string
|
||||
// the frontend always passes a continuation token
|
||||
token := aws.String("")
|
||||
for {
|
||||
out, err := p.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: &bucket, Prefix: &prefix, Delimiter: &delim,
|
||||
StartAfter: &startAfter, ContinuationToken: token, MaxKeys: aws.Int32(2),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
for _, obj := range out.Contents {
|
||||
contents = append(contents, *obj.Key)
|
||||
}
|
||||
for _, cp := range out.CommonPrefixes {
|
||||
prefixes = append(prefixes, *cp.Prefix)
|
||||
}
|
||||
if !*out.IsTruncated {
|
||||
return contents, prefixes
|
||||
}
|
||||
token = out.NextContinuationToken
|
||||
}
|
||||
}
|
||||
checkList := func(got, want []string) {
|
||||
t.Helper()
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("list: got %+q, want %+q", got, want)
|
||||
}
|
||||
}
|
||||
contents, _ := list("", "", "")
|
||||
checkList(contents, sorted)
|
||||
contents, _ = list("sort/a?", "", "")
|
||||
checkList(contents, []string{"sort/a?", "sort/a?b"})
|
||||
contents, prefixes := list("sort/", "/", "")
|
||||
checkList(contents, []string{"sort/a", "sort/a!", "sort/a.", "sort/a:z", "sort/a?", "sort/a?b", "sort/aA", "sort/a\uf03f"})
|
||||
checkList(prefixes, []string{"sort/a*/"})
|
||||
contents, prefixes = list("sort/", "?", "")
|
||||
checkList(contents, []string{"sort/a", "sort/a!", "sort/a*/y", "sort/a.", "sort/a:z", "sort/aA", "sort/a\uf03f"})
|
||||
checkList(prefixes, []string{"sort/a?"})
|
||||
contents, _ = list("sort/", "", "sort/a?")
|
||||
checkList(contents, []string{"sort/a?b", "sort/aA", "sort/a\uf03f"})
|
||||
v1, err := p.ListObjects(ctx, &s3.ListObjectsInput{
|
||||
Bucket: &bucket, Prefix: aws.String("sort/"), Marker: aws.String("sort/a."), MaxKeys: aws.Int32(1000),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list v1: %v", err)
|
||||
}
|
||||
contents = nil
|
||||
for _, obj := range v1.Contents {
|
||||
contents = append(contents, *obj.Key)
|
||||
}
|
||||
checkList(contents, []string{"sort/a:z", "sort/a?", "sort/a?b", "sort/aA", "sort/a\uf03f"})
|
||||
|
||||
if err := p.PutObjectTagging(ctx, bucket, "question?.txt", "", map[string]string{"k": "v"}); err != nil {
|
||||
t.Fatalf("put tagging: %v", err)
|
||||
}
|
||||
tags, err := p.GetObjectTagging(ctx, bucket, "question?.txt", "")
|
||||
if err != nil || tags["k"] != "v" {
|
||||
t.Errorf("get tagging: got %v, %v", tags, err)
|
||||
}
|
||||
|
||||
copyKey := "copy*dst?."
|
||||
_, err = p.CopyObject(ctx, s3response.CopyObjectInput{
|
||||
Bucket: &bucket, Key: ©Key, CopySource: aws.String(bucket + "/" + "question?.txt"),
|
||||
ExpectedBucketOwner: aws.String(""), MetadataDirective: types.MetadataDirectiveCopy,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("copy: %v", err)
|
||||
}
|
||||
if data, _ := getTestObject(t, p, bucket, copyKey); string(data) != "question?.txt" {
|
||||
t.Errorf("get copy: got %+q", data)
|
||||
}
|
||||
|
||||
mpKey := `mp?*:<>|"\.`
|
||||
mp, err := p.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{Bucket: &bucket, Key: &mpKey})
|
||||
if err != nil {
|
||||
t.Fatalf("create mp: %v", err)
|
||||
}
|
||||
uploads, err := p.ListMultipartUploads(ctx, &s3.ListMultipartUploadsInput{
|
||||
Bucket: &bucket, Prefix: aws.String("mp?"), MaxUploads: aws.Int32(1000),
|
||||
})
|
||||
if err != nil || len(uploads.Uploads) != 1 || uploads.Uploads[0].Key != mpKey {
|
||||
t.Errorf("list mp uploads: got %+v, %v", uploads.Uploads, err)
|
||||
}
|
||||
part, err := p.UploadPart(ctx, &s3.UploadPartInput{
|
||||
Bucket: &bucket, Key: &mpKey, UploadId: &mp.UploadId, PartNumber: aws.Int32(1),
|
||||
Body: strings.NewReader("part"), ContentLength: aws.Int64(4),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("upload part: %v", err)
|
||||
}
|
||||
done, _, err := p.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: &bucket, Key: &mpKey, UploadId: &mp.UploadId,
|
||||
MultipartUpload: &types.CompletedMultipartUpload{
|
||||
Parts: []types.CompletedPart{{PartNumber: aws.Int32(1), ETag: part.ETag}},
|
||||
},
|
||||
})
|
||||
if err != nil || *done.Key != mpKey {
|
||||
t.Fatalf("complete mp: got %v, %v", done.Key, err)
|
||||
}
|
||||
if data, _ := getTestObject(t, p, bucket, mpKey); string(data) != "part" {
|
||||
t.Errorf("get mp object: got %+q", data)
|
||||
}
|
||||
|
||||
copyPartKey := "mp2*"
|
||||
mp2, err := p.CreateMultipartUpload(ctx, s3response.CreateMultipartUploadInput{Bucket: &bucket, Key: ©PartKey})
|
||||
if err != nil {
|
||||
t.Fatalf("create mp: %v", err)
|
||||
}
|
||||
_, err = p.UploadPartCopy(ctx, &s3.UploadPartCopyInput{
|
||||
Bucket: &bucket, Key: ©PartKey, UploadId: &mp2.UploadId, PartNumber: aws.Int32(1),
|
||||
CopySource: aws.String(bucket + "/" + "question?.txt"), CopySourceRange: aws.String(""),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("upload part copy: %v", err)
|
||||
}
|
||||
err = p.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{Bucket: &bucket, Key: ©PartKey, UploadId: &mp2.UploadId})
|
||||
if err != nil {
|
||||
t.Fatalf("abort mp: %v", err)
|
||||
}
|
||||
|
||||
if got := p.NormalizeObjectKey(bucket, `..\..\escape`); got != `..\..\escape` {
|
||||
t.Errorf("normalize: got %+q", got)
|
||||
}
|
||||
|
||||
if err := p.PutBucketVersioning(ctx, bucket, types.BucketVersioningStatusEnabled); err != nil {
|
||||
t.Fatalf("put bucket versioning: %v", err)
|
||||
}
|
||||
if err := p.PutObjectLockConfiguration(ctx, bucket, []byte(`{"Enabled":true}`)); err != nil {
|
||||
t.Fatalf("put object lock configuration: %v", err)
|
||||
}
|
||||
lockKey := "nul.txt"
|
||||
if err := p.PutObjectLegalHold(ctx, bucket, lockKey, "", true); err != nil {
|
||||
t.Fatalf("put legal hold: %v", err)
|
||||
}
|
||||
head, err := p.HeadObject(ctx, &s3.HeadObjectInput{Bucket: &bucket, Key: &lockKey})
|
||||
if err != nil || head.ObjectLockLegalHoldStatus != types.ObjectLockLegalHoldStatusOn {
|
||||
t.Errorf("head legal hold: got %+v, %v", head, err)
|
||||
}
|
||||
retention := []byte(`{"Mode":"GOVERNANCE"}`)
|
||||
if err := p.PutObjectRetention(ctx, bucket, lockKey, "", retention); err != nil {
|
||||
t.Fatalf("put retention: %v", err)
|
||||
}
|
||||
if got, err := p.GetObjectRetention(ctx, bucket, lockKey, ""); err != nil || string(got) != string(retention) {
|
||||
t.Errorf("get retention: got %s, %v", got, err)
|
||||
}
|
||||
|
||||
vKey := "trailingdot."
|
||||
if err := putTestObject(t, p, bucket, vKey, "v2"); err != nil {
|
||||
t.Fatalf("put version: %v", err)
|
||||
}
|
||||
if _, err := p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &vKey}); err != nil {
|
||||
t.Fatalf("delete: %v", err)
|
||||
}
|
||||
versions, err := p.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
|
||||
Bucket: &bucket, Prefix: &vKey, MaxKeys: aws.Int32(1000),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list versions: %v", err)
|
||||
}
|
||||
if len(versions.DeleteMarkers) != 1 || *versions.DeleteMarkers[0].Key != vKey || len(versions.Versions) != 2 {
|
||||
t.Fatalf("list versions: got %+v", versions)
|
||||
}
|
||||
for _, v := range versions.Versions {
|
||||
out, err := p.GetObject(ctx, &s3.GetObjectInput{Bucket: &bucket, Key: &vKey, VersionId: v.VersionId})
|
||||
if err != nil {
|
||||
t.Fatalf("get version: %v", err)
|
||||
}
|
||||
data, _ := io.ReadAll(out.Body)
|
||||
out.Body.Close()
|
||||
if *v.Key != vKey || (string(data) != vKey && string(data) != "v2") {
|
||||
t.Errorf("get version %v of %+q: got %+q", *v.VersionId, *v.Key, data)
|
||||
}
|
||||
}
|
||||
_, err = p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &vKey, VersionId: versions.DeleteMarkers[0].VersionId})
|
||||
if err != nil {
|
||||
t.Fatalf("delete marker: %v", err)
|
||||
}
|
||||
if data, _ := getTestObject(t, p, bucket, vKey); string(data) != "v2" {
|
||||
t.Errorf("get restored version: got %+q", data)
|
||||
}
|
||||
|
||||
for _, dir := range []string{root, versionDir, opts.SideCarDir} {
|
||||
if dir == "" {
|
||||
continue
|
||||
}
|
||||
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || path == dir {
|
||||
return err
|
||||
}
|
||||
checkWindowsName(t, d.Name())
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("walk %v: %v", dir, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range append(keys, copyKey, mpKey) {
|
||||
_, err := p.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &key})
|
||||
if err != nil {
|
||||
t.Errorf("delete %+q: %v", key, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// checkWindowsName fails the test if the path elements of name can't name a
|
||||
// file on Windows, or on Windows 10 when not run on Windows.
|
||||
func checkWindowsName(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
for _, elem := range strings.Split(name, "/") {
|
||||
if elem == "" || elem == "." || elem == ".." {
|
||||
continue
|
||||
}
|
||||
if strings.ContainsFunc(elem, isInvalidWindowsChar) ||
|
||||
strings.HasSuffix(elem, ".") || strings.HasSuffix(elem, " ") ||
|
||||
isReservedName(elem) {
|
||||
t.Errorf("%+q isn't a valid Windows file name", elem)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright 2026 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build windows
|
||||
|
||||
package posix
|
||||
|
||||
import "path/filepath"
|
||||
|
||||
// encodeKeysDefault stores the keys under names that are valid on Windows.
|
||||
const encodeKeysDefault = true
|
||||
|
||||
// isReservedName reports whether Windows opens a device for the path element
|
||||
// name, which has no path separator or colon. A device name with an extension,
|
||||
// such as NUL.txt, is a device on Windows 10 but not on Windows 11, which
|
||||
// filepath.IsLocal asks the system about.
|
||||
func isReservedName(name string) bool {
|
||||
return !filepath.IsLocal(name)
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright 2026 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build windows
|
||||
|
||||
package posix
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestReservedNameWithoutExtension checks that a device name without an
|
||||
// extension is reserved on every version of Windows.
|
||||
func TestReservedNameWithoutExtension(t *testing.T) {
|
||||
for _, name := range []string{"CON", "con", "nul ", "AUX", "PRN", "COM1", "lpt9", "COM\u00b9", "CONIN$", "conout$"} {
|
||||
if !isReservedName(name) {
|
||||
t.Errorf("%+q isn't reserved", name)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"CONSOLE", "COM0", "LPT", "a", "con\uf020"} {
|
||||
if isReservedName(name) {
|
||||
t.Errorf("%+q is reserved", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEncodedNamesAreFiles checks that Windows stores a file under the name
|
||||
// of every key, apart from the keys it would otherwise store it as.
|
||||
func TestEncodedNamesAreFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
keys := []string{
|
||||
"question?", "question", "a:b", "a", `back\slash`, "pipe|<>\"*", "tab\tchar",
|
||||
"trailingdot.", "trailingdot", "trailingspace ", "trailingspace",
|
||||
"CON", "nul.txt", "COM1", "LPT\u00b9", "CONIN$", "pua\uf03f", "pua?",
|
||||
}
|
||||
for _, key := range keys {
|
||||
name := filepath.Join(dir, encodeWindowsName(key, isReservedName))
|
||||
if err := os.WriteFile(name, []byte(key), 0644); err != nil {
|
||||
t.Fatalf("write %+q: %v", key, err)
|
||||
}
|
||||
}
|
||||
for _, key := range keys {
|
||||
data, err := os.ReadFile(filepath.Join(dir, encodeWindowsName(key, isReservedName)))
|
||||
if err != nil || string(data) != key {
|
||||
t.Errorf("read %+q: got %+q, %v", key, data, err)
|
||||
}
|
||||
}
|
||||
|
||||
ents, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for _, ent := range ents {
|
||||
got = append(got, decodeWindowsName(ent.Name()))
|
||||
}
|
||||
slices.Sort(got)
|
||||
slices.Sort(keys)
|
||||
if !slices.Equal(got, keys) {
|
||||
t.Errorf("read dir: got %+q, want %+q", got, keys)
|
||||
}
|
||||
}
|
||||
+79
-60
@@ -114,6 +114,10 @@ type Posix struct {
|
||||
// frontend handlers are already validating bucket names
|
||||
validateBucketName bool
|
||||
|
||||
// encodeKeys stores the objects under names that are valid on Windows,
|
||||
// see encodeKey. It's set on Windows.
|
||||
encodeKeys bool
|
||||
|
||||
// defaultEtag is returned for objects that do not have an etag attribute
|
||||
// stored (e.g. files placed on the filesystem outside of versitygw).
|
||||
defaultEtag string
|
||||
@@ -427,6 +431,7 @@ func New(rootdir string, ms meta.MetadataStorer, opts PosixOpts) (*Posix, error)
|
||||
objectLockMode: objectLockMode,
|
||||
enableODirect: opts.EnableODirect,
|
||||
validateBucketName: opts.ValidateBucketNames,
|
||||
encodeKeys: encodeKeysDefault,
|
||||
actionLimiter: semaphore.NewWeighted(int64(concurrencyOrDefault(opts.Concurrency))),
|
||||
copyObjectThreshold: opts.CopyObjectThreshold,
|
||||
defaultEtag: opts.DefaultEtag,
|
||||
@@ -1596,8 +1601,7 @@ func (p *Posix) ListObjectVersions(ctx context.Context, input *s3.ListObjectVers
|
||||
return s3response.ListVersionsResult{}, err
|
||||
}
|
||||
|
||||
fileSystem := os.DirFS(p.BucketPath(bucket))
|
||||
results, err := backend.WalkVersions(ctx, fileSystem, prefix, delim, keyMarker, versionIdMarker, max,
|
||||
results, err := backend.WalkVersions(ctx, p.bucketFS(bucket), prefix, delim, keyMarker, versionIdMarker, max,
|
||||
p.fileToObjVersions(bucket), []string{MetaTmpDir})
|
||||
if err != nil {
|
||||
return s3response.ListVersionsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
@@ -1712,6 +1716,8 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
|
||||
if d.IsDir() {
|
||||
key = path + "/"
|
||||
}
|
||||
// the walk is over the keys, the object is stored at the encoded key
|
||||
path = p.encodeKey(path)
|
||||
|
||||
// get object info and fill out object data
|
||||
etagBytes, err := p.meta.RetrieveAttribute(nil, bucket, path, etagkey)
|
||||
@@ -1805,7 +1811,7 @@ func (p *Posix) fileToObjVersions(bucket string) backend.GetVersionsFunc {
|
||||
}
|
||||
|
||||
// List all the versions of the object in the versioning directory
|
||||
versionPath := p.genObjVersionPath(bucket, key)
|
||||
versionPath := p.genObjVersionPath(bucket, p.encodeKey(key))
|
||||
dirEnts, err := os.ReadDir(versionPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return &backend.ObjVersionFuncResult{
|
||||
@@ -2344,7 +2350,8 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
|
||||
}
|
||||
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
key := *input.Key
|
||||
object := p.encodeKey(key)
|
||||
uploadID := *input.UploadId
|
||||
parts := input.MultipartUpload.Parts
|
||||
|
||||
@@ -2365,7 +2372,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
|
||||
// the processing slot. A concurrent call with the same uploadId will compute
|
||||
// the same ETag, so it will either find the directory still present (still
|
||||
// processing) or gone (already completed) and react accordingly.
|
||||
sum := sha256.Sum256([]byte(object))
|
||||
sum := sha256.Sum256([]byte(key))
|
||||
objdirFull := filepath.Join(p.BucketPath(bucket), MetaTmpMultipartDir, fmt.Sprintf("%x", sum))
|
||||
uploadIDDir := filepath.Join(objdirFull, uploadID)
|
||||
// Compute the default multipart ETag token used for claim naming.
|
||||
@@ -2487,7 +2494,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
|
||||
return s3response.CompleteMultipartUploadResult{
|
||||
Bucket: &bucket,
|
||||
ETag: &etag,
|
||||
Key: &object,
|
||||
Key: &key,
|
||||
}, "", nil
|
||||
}
|
||||
// Directory is gone: a completed upload is linked before its
|
||||
@@ -2504,7 +2511,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
|
||||
return res, "", s3err.GetAPIError(s3err.ErrNoSuchUpload)
|
||||
}
|
||||
if isErrNameTooLong(err) {
|
||||
return res, "", s3err.GetKeyTooLongErr(int64(len(object)), 1024)
|
||||
return res, "", s3err.GetKeyTooLongErr(int64(len(key)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
return res, "", fmt.Errorf("stat object: %w", err)
|
||||
@@ -2547,7 +2554,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
|
||||
return s3response.CompleteMultipartUploadResult{
|
||||
Bucket: &bucket,
|
||||
ETag: &etag,
|
||||
Key: &object,
|
||||
Key: &key,
|
||||
}, string(versionID), nil
|
||||
}
|
||||
if err != nil {
|
||||
@@ -3028,7 +3035,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C
|
||||
return s3response.CompleteMultipartUploadResult{
|
||||
Bucket: &bucket,
|
||||
ETag: &finalEtag,
|
||||
Key: &object,
|
||||
Key: &key,
|
||||
ChecksumCRC32: crc32,
|
||||
ChecksumCRC32C: crc32c,
|
||||
ChecksumSHA1: sha1,
|
||||
@@ -4146,10 +4153,11 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
|
||||
|
||||
partPath := filepath.Join(objdir, *upi.UploadId, fmt.Sprintf("%v", *upi.PartNumber))
|
||||
|
||||
srcBucket, srcObject, srcVersionId, err := backend.ParseCopySource(*upi.CopySource)
|
||||
srcBucket, srcKey, srcVersionId, err := backend.ParseCopySource(*upi.CopySource)
|
||||
if err != nil {
|
||||
return s3response.CopyPartResult{}, err
|
||||
}
|
||||
srcObject := p.encodeKey(srcKey)
|
||||
if err := p.validateVersionId(srcVersionId); err != nil {
|
||||
return s3response.CopyPartResult{}, err
|
||||
}
|
||||
@@ -4204,12 +4212,12 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
|
||||
fi, err := os.Stat(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
if p.versioningEnabled() && vEnabled {
|
||||
return s3response.CopyPartResult{}, s3err.GetNoSuchVersionErr(srcObject, srcVersionId)
|
||||
return s3response.CopyPartResult{}, s3err.GetNoSuchVersionErr(srcKey, srcVersionId)
|
||||
}
|
||||
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if isErrNameTooLong(err) {
|
||||
return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(srcObject)), 1024)
|
||||
return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(srcKey)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
return s3response.CopyPartResult{}, fmt.Errorf("stat object: %w", err)
|
||||
@@ -4561,13 +4569,14 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
|
||||
name := p.ObjectPath(*po.Bucket, *po.Key)
|
||||
object := p.encodeKey(*po.Key)
|
||||
name := p.ObjectPath(*po.Bucket, object)
|
||||
|
||||
// Fast-fail precondition check before the request body is staged. This
|
||||
// is only advisory: the authoritative check is repeated while holding
|
||||
// the object publish lock at commit time so that evaluation and
|
||||
// publication are atomic under concurrent writers.
|
||||
err = p.checkPutPreconditions(*po.Bucket, *po.Key, po.IfMatch, po.IfNoneMatch)
|
||||
err = p.checkPutPreconditions(*po.Bucket, object, po.IfMatch, po.IfNoneMatch)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
@@ -4616,7 +4625,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
}
|
||||
}
|
||||
|
||||
if strings.HasSuffix(*po.Key, "/") {
|
||||
if strings.HasSuffix(object, "/") {
|
||||
// object is directory
|
||||
if contentLength != 0 {
|
||||
// posix directories can't contain data, send error
|
||||
@@ -4627,13 +4636,13 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
|
||||
// serialize with competing writers to the same key and repeat the
|
||||
// precondition check authoritatively under the publish lock
|
||||
unlock, err := p.lockObjectPublish(ctx, *po.Bucket, *po.Key)
|
||||
unlock, err := p.lockObjectPublish(ctx, *po.Bucket, object)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
err = p.checkPutPreconditions(*po.Bucket, *po.Key, po.IfMatch, po.IfNoneMatch)
|
||||
err = p.checkPutPreconditions(*po.Bucket, object, po.IfMatch, po.IfNoneMatch)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
@@ -4663,7 +4672,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
|
||||
// In a versioned bucket the directory is kept across versions: its
|
||||
// current version is copied to the versioning directory first.
|
||||
err = p.snapshotObjVersion(*po.Bucket, *po.Key, vStatus, acct)
|
||||
err = p.snapshotObjVersion(*po.Bucket, object, vStatus, acct)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
@@ -4682,7 +4691,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
// The directory outlives the object it replaces, whether or not
|
||||
// the bucket is versioned: the new object starts with none of the
|
||||
// replaced object's attributes.
|
||||
err = p.clearDirObjectAttrs(*po.Bucket, *po.Key)
|
||||
err = p.clearDirObjectAttrs(*po.Bucket, object)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
@@ -4690,7 +4699,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
var versionID string
|
||||
if versioned {
|
||||
if p.isBucketVersioningSuspended(vStatus) {
|
||||
err = p.deleteNullVersionIdObject(*po.Bucket, *po.Key)
|
||||
err = p.deleteNullVersionIdObject(*po.Bucket, object)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
@@ -4699,7 +4708,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
}
|
||||
}
|
||||
|
||||
err = p.storeObjectMetadata(nil, *po.Bucket, *po.Key, po.Metadata)
|
||||
err = p.storeObjectMetadata(nil, *po.Bucket, object, po.Metadata)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("set object metadata: %w", err)
|
||||
}
|
||||
@@ -4710,21 +4719,21 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
}
|
||||
|
||||
// set etag attribute to signify this dir was specifically put
|
||||
err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, etagkey,
|
||||
err = p.meta.StoreAttribute(nil, *po.Bucket, object, etagkey,
|
||||
[]byte(dirETag))
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err)
|
||||
}
|
||||
|
||||
// set "application/x-directory" content-type
|
||||
err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, contentTypeHdr,
|
||||
err = p.meta.StoreAttribute(nil, *po.Bucket, object, contentTypeHdr,
|
||||
[]byte(backend.DirContentType))
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("set content-type attr: %w", err)
|
||||
}
|
||||
|
||||
if getString(po.WebsiteRedirectLocation) != "" {
|
||||
err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, websiteRedirectHdr,
|
||||
err = p.meta.StoreAttribute(nil, *po.Bucket, object, websiteRedirectHdr,
|
||||
[]byte(*po.WebsiteRedirectLocation))
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("set website-redirect-location attr: %w", err)
|
||||
@@ -4739,13 +4748,13 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
|
||||
checksum.SetSum(checksumAlgorithm, &expectedSum)
|
||||
|
||||
err = p.storeChecksums(nil, *po.Bucket, *po.Key, checksum)
|
||||
err = p.storeChecksums(nil, *po.Bucket, object, checksum)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err)
|
||||
}
|
||||
|
||||
if versionID != "" {
|
||||
err = p.meta.StoreAttribute(nil, *po.Bucket, *po.Key, versionIdKey, []byte(versionID))
|
||||
err = p.meta.StoreAttribute(nil, *po.Bucket, object, versionIdKey, []byte(versionID))
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err)
|
||||
}
|
||||
@@ -4819,7 +4828,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
}
|
||||
|
||||
f, err := p.openTmpFile(p.ObjectPath(*po.Bucket, MetaTmpDir),
|
||||
*po.Bucket, *po.Key, contentLength, acct, doFalloc, p.forceNoTmpFile, odirectAllowed)
|
||||
*po.Bucket, object, contentLength, acct, doFalloc, p.forceNoTmpFile, odirectAllowed)
|
||||
if err != nil {
|
||||
if errors.Is(err, syscall.EDQUOT) {
|
||||
drainBody(po.Body)
|
||||
@@ -4917,7 +4926,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
// The body is fully staged; serialize the commit phase (condition
|
||||
// evaluation, metadata stores, and final link) with competing writers
|
||||
// to the same key.
|
||||
unlock, err := p.lockObjectPublish(ctx, *po.Bucket, *po.Key)
|
||||
unlock, err := p.lockObjectPublish(ctx, *po.Bucket, object)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
@@ -4925,7 +4934,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
|
||||
if conditional {
|
||||
// authoritative precondition check under the publish lock
|
||||
err = p.checkPutPreconditions(*po.Bucket, *po.Key, po.IfMatch, po.IfNoneMatch)
|
||||
err = p.checkPutPreconditions(*po.Bucket, object, po.IfMatch, po.IfNoneMatch)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
@@ -4941,7 +4950,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
|
||||
// Snapshot the object that is about to be replaced only after the
|
||||
// conditional check has succeeded and while holding the publish lock.
|
||||
verr := p.snapshotObjVersion(*po.Bucket, *po.Key, vStatus, acct)
|
||||
verr := p.snapshotObjVersion(*po.Bucket, object, vStatus, acct)
|
||||
if verr != nil {
|
||||
return s3response.PutObjectOutput{}, verr
|
||||
}
|
||||
@@ -4953,7 +4962,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
if versionID != "" {
|
||||
keep = append(keep, versionIdKey)
|
||||
}
|
||||
err = p.meta.ReplaceObject(*po.Bucket, *po.Key, keep)
|
||||
err = p.meta.ReplaceObject(*po.Bucket, object, keep)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("remove replaced object attributes: %w", err)
|
||||
}
|
||||
@@ -4962,7 +4971,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
// null versionId object from versioning directory
|
||||
// if it exists and the versioning status is Suspended
|
||||
if p.isBucketVersioningSuspended(vStatus) {
|
||||
err = p.deleteNullVersionIdObject(*po.Bucket, *po.Key)
|
||||
err = p.deleteNullVersionIdObject(*po.Bucket, object)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, err
|
||||
}
|
||||
@@ -4995,17 +5004,17 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
}
|
||||
|
||||
checksum.SetSum(checksumAlgorithm, &sum)
|
||||
err = p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum)
|
||||
err = p.storeChecksums(f.File(), *po.Bucket, object, checksum)
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("store checksum: %w", err)
|
||||
}
|
||||
|
||||
err = p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, etagkey, []byte(etag))
|
||||
err = p.meta.StoreAttribute(f.File(), *po.Bucket, object, etagkey, []byte(etag))
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("set etag attr: %w", err)
|
||||
}
|
||||
|
||||
err = p.storeObjectMetaProperties(f.File(), *po.Bucket, *po.Key,
|
||||
err = p.storeObjectMetaProperties(f.File(), *po.Bucket, object,
|
||||
metaProperties{
|
||||
ContentType: po.ContentType,
|
||||
ContentEncoding: po.ContentEncoding,
|
||||
@@ -5021,7 +5030,7 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
}
|
||||
|
||||
if versionID != "" && versionID != nullVersionId {
|
||||
err := p.meta.StoreAttribute(f.File(), *po.Bucket, *po.Key, versionIdKey, []byte(versionID))
|
||||
err := p.meta.StoreAttribute(f.File(), *po.Bucket, object, versionIdKey, []byte(versionID))
|
||||
if err != nil {
|
||||
return s3response.PutObjectOutput{}, fmt.Errorf("set versionId attr: %w", err)
|
||||
}
|
||||
@@ -5142,7 +5151,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
}
|
||||
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
object := p.encodeKey(*input.Key)
|
||||
isDir := strings.HasSuffix(object, "/")
|
||||
|
||||
if !p.isBucketValid(bucket) {
|
||||
@@ -5198,7 +5207,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
return &s3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
@@ -5284,7 +5293,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil
|
||||
}
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat object: %w", err)
|
||||
@@ -5452,7 +5461,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
|
||||
err = os.Remove(filepath.Join(versionPath, *input.VersionId))
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
|
||||
}
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, *input.VersionId)
|
||||
@@ -5480,7 +5489,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
|
||||
fi, err := os.Stat(objpath)
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
|
||||
}
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
// AWS returns success if the object does not exist
|
||||
@@ -5840,7 +5849,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
return nil, err
|
||||
}
|
||||
|
||||
object := *input.Key
|
||||
object := p.encodeKey(*input.Key)
|
||||
if versionId != "" {
|
||||
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
@@ -6179,7 +6188,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
|
||||
}
|
||||
|
||||
bucket := *input.Bucket
|
||||
object := *input.Key
|
||||
object := p.encodeKey(*input.Key)
|
||||
|
||||
if !p.isBucketValid(bucket) {
|
||||
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
|
||||
@@ -6349,7 +6358,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
|
||||
}
|
||||
|
||||
var objectLockLegalHoldStatus types.ObjectLockLegalHoldStatus
|
||||
status, err := p.GetObjectLegalHold(withCtxNoSlot(ctx), bucket, object, versionId)
|
||||
status, err := p.GetObjectLegalHold(withCtxNoSlot(ctx), bucket, *input.Key, versionId)
|
||||
if err == nil {
|
||||
if *status {
|
||||
objectLockLegalHoldStatus = types.ObjectLockLegalHoldStatusOn
|
||||
@@ -6360,7 +6369,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
|
||||
|
||||
var objectLockMode types.ObjectLockMode
|
||||
var objectLockRetainUntilDate *time.Time
|
||||
retention, err := p.GetObjectRetention(withCtxNoSlot(ctx), bucket, object, versionId)
|
||||
retention, err := p.GetObjectRetention(withCtxNoSlot(ctx), bucket, *input.Key, versionId)
|
||||
if err == nil {
|
||||
var config types.ObjectLockRetention
|
||||
if err := json.Unmarshal(retention, &config); err == nil {
|
||||
@@ -6506,7 +6515,8 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
|
||||
return s3response.CopyObjectOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, srcBucket)
|
||||
}
|
||||
dstBucket := *input.Bucket
|
||||
dstObject := *input.Key
|
||||
dstKey := *input.Key
|
||||
dstObject := p.encodeKey(dstKey)
|
||||
|
||||
if !p.isBucketValid(dstBucket) {
|
||||
return s3response.CopyObjectOutput{}, s3err.GetBucketErr(s3err.ErrInvalidBucketName, dstBucket)
|
||||
@@ -6552,6 +6562,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
|
||||
vEnabled := p.isBucketVersioningEnabled(vStatus)
|
||||
|
||||
origSrcObject := srcObject
|
||||
srcObject = p.encodeKey(srcObject)
|
||||
if srcVersionId != "" {
|
||||
if !p.versioningEnabled() || !vEnabled {
|
||||
return s3response.CopyObjectOutput{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, srcVersionId)
|
||||
@@ -6792,7 +6803,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
|
||||
err = p.PutObjectTagging(withCtxNoSlot(ctx), dstBucket, dstObject, "", tags)
|
||||
err = p.PutObjectTagging(withCtxNoSlot(ctx), dstBucket, dstKey, "", tags)
|
||||
if err != nil {
|
||||
return s3response.CopyObjectOutput{}, err
|
||||
}
|
||||
@@ -6802,7 +6813,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
|
||||
// settings of a newly written one
|
||||
err = p.putObjectLockSettings(ctx, s3response.PutObjectInput{
|
||||
Bucket: &dstBucket,
|
||||
Key: &dstObject,
|
||||
Key: &dstKey,
|
||||
ObjectLockRetainUntilDate: input.ObjectLockRetainUntilDate,
|
||||
ObjectLockMode: input.ObjectLockMode,
|
||||
ObjectLockLegalHoldStatus: input.ObjectLockLegalHoldStatus,
|
||||
@@ -6834,7 +6845,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
|
||||
|
||||
putObjectInput := s3response.PutObjectInput{
|
||||
Bucket: &dstBucket,
|
||||
Key: &dstObject,
|
||||
Key: &dstKey,
|
||||
Body: srcBody,
|
||||
ContentLength: &contentLength,
|
||||
ChecksumAlgorithm: checksums.Algorithm,
|
||||
@@ -6975,8 +6986,7 @@ func (p *Posix) ListObjectsParametrized(ctx context.Context, input *s3.ListObjec
|
||||
return s3response.ListObjectsResult{}, err
|
||||
}
|
||||
|
||||
fileSystem := os.DirFS(p.BucketPath(bucket))
|
||||
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
|
||||
results, err := backend.Walk(ctx, p.bucketFS(bucket), prefix, delim, marker, maxkeys,
|
||||
customFileToObj(bucket, true), []string{MetaTmpDir})
|
||||
if err != nil {
|
||||
return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
@@ -7002,7 +7012,10 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc {
|
||||
}
|
||||
}
|
||||
|
||||
return func(path string, d fs.DirEntry) (s3response.Object, error) {
|
||||
return func(key string, d fs.DirEntry) (s3response.Object, error) {
|
||||
// the walk is over the keys, the object is stored at the encoded key
|
||||
path := p.encodeKey(key)
|
||||
|
||||
var owner *types.Owner
|
||||
// Retrieve the object owner data from bucket ACL, if fetchOwner is true
|
||||
// All the objects in the bucket are owned by the bucket owner
|
||||
@@ -7052,7 +7065,7 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc {
|
||||
|
||||
return s3response.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
Key: &key,
|
||||
LastModified: &mtime,
|
||||
Size: &size,
|
||||
StorageClass: types.ObjectStorageClassStandard,
|
||||
@@ -7101,7 +7114,7 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc {
|
||||
|
||||
return s3response.Object{
|
||||
ETag: &etag,
|
||||
Key: &path,
|
||||
Key: &key,
|
||||
LastModified: &mtime,
|
||||
Size: &size,
|
||||
StorageClass: types.ObjectStorageClassStandard,
|
||||
@@ -7158,8 +7171,7 @@ func (p *Posix) ListObjectsV2Parametrized(ctx context.Context, input *s3.ListObj
|
||||
return s3response.ListObjectsV2Result{}, err
|
||||
}
|
||||
|
||||
fileSystem := os.DirFS(p.BucketPath(bucket))
|
||||
results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys,
|
||||
results, err := backend.Walk(ctx, p.bucketFS(bucket), prefix, delim, marker, maxkeys,
|
||||
customFileToObj(bucket, fetchOwner), []string{MetaTmpDir})
|
||||
if err != nil {
|
||||
return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err)
|
||||
@@ -7325,6 +7337,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
|
||||
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
|
||||
}
|
||||
|
||||
object = p.encodeKey(object)
|
||||
bucket, object, err = p.objVersionAttrPath(bucket, object, versionId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -7392,6 +7405,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
|
||||
return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
|
||||
}
|
||||
|
||||
object = p.encodeKey(object)
|
||||
bucket, object, err = p.objVersionAttrPath(bucket, object, versionId)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -7826,6 +7840,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI
|
||||
if !p.isBucketValid(bucket) {
|
||||
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
|
||||
}
|
||||
object = p.encodeKey(object)
|
||||
err = p.doesBucketAndObjectExist(bucket, object, versionId)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -7886,6 +7901,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI
|
||||
if !p.isBucketValid(bucket) {
|
||||
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
|
||||
}
|
||||
object = p.encodeKey(object)
|
||||
err = p.doesBucketAndObjectExist(bucket, object, versionId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -7946,6 +7962,7 @@ func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionI
|
||||
if !p.isBucketValid(bucket) {
|
||||
return s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
|
||||
}
|
||||
object = p.encodeKey(object)
|
||||
err = p.doesBucketAndObjectExist(bucket, object, versionId)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -7993,6 +8010,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI
|
||||
if !p.isBucketValid(bucket) {
|
||||
return nil, s3err.GetBucketErr(s3err.ErrInvalidBucketName, bucket)
|
||||
}
|
||||
object = p.encodeKey(object)
|
||||
err = p.doesBucketAndObjectExist(bucket, object, versionId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -8133,18 +8151,19 @@ func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.
|
||||
// attempt that escapes bucket), but the result names an S3 key, not a host
|
||||
// path: on Windows filepath.Join/Rel would return it with "\" separators,
|
||||
// which callers building a policy-resource ARN or match string must never
|
||||
// see, so it's converted back to "/" before returning.
|
||||
// see, so it's converted back to "/" before returning. The key is resolved
|
||||
// as the path it's stored at, in which a "\" of the key is no separator.
|
||||
func (p *Posix) NormalizeObjectKey(bucket, object string) string {
|
||||
fullPath := filepath.Join(bucket, object)
|
||||
fullPath := filepath.Join(bucket, p.encodeKey(object))
|
||||
key, err := filepath.Rel(filepath.Clean(bucket), fullPath)
|
||||
if err != nil {
|
||||
return filepath.ToSlash(fullPath)
|
||||
return p.decodeKey(filepath.ToSlash(fullPath))
|
||||
}
|
||||
if key == "." {
|
||||
return ""
|
||||
}
|
||||
|
||||
return filepath.ToSlash(key)
|
||||
return p.decodeKey(filepath.ToSlash(key))
|
||||
}
|
||||
|
||||
func (p *Posix) storeChecksums(f *os.File, bucket, object string, chs s3response.Checksum) error {
|
||||
|
||||
@@ -53,21 +53,10 @@ func PutObject_special_chars(s *S3Conf) error {
|
||||
"my?key", "my^key", "my{}key", "my%key", "my`key",
|
||||
"my[]key", "my~key", "my<>key", "my|key", "my#key",
|
||||
}
|
||||
if !s.azureTests && !s.windowsTests {
|
||||
// azure and windows cannot handle backslashes in object names:
|
||||
// on Windows, '\' is a path separator so 'my\key' is stored as 'my/key'
|
||||
if !s.azureTests {
|
||||
// azure cannot handle backslashes in object names
|
||||
objnames = append(objnames, "my\\key")
|
||||
}
|
||||
if s.windowsTests {
|
||||
// ':', '?', '<', '>', '|', '*' are not valid filename characters on Windows
|
||||
var filtered []string
|
||||
for _, name := range objnames {
|
||||
if !strings.ContainsAny(name, `:?<>|*`) {
|
||||
filtered = append(filtered, name)
|
||||
}
|
||||
}
|
||||
objnames = filtered
|
||||
}
|
||||
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
objs, err := putObjects(s3client, objnames, bucket)
|
||||
@@ -1731,21 +1720,13 @@ func PutObject_false_negative_object_names(s *S3Conf) error {
|
||||
"..%2f..%2f..%2fsecret/file.txt", // URL-encoded
|
||||
"..%c0%af..%c0%afetc/passwd", // UTF-8 overlong trick
|
||||
"dir/%c0%ae%c0%ae/%c0%ae%c0%ae/etc/passwd", // overlong UTF-8 encoding
|
||||
".../.../.../target.txt",
|
||||
"dir/%20../file.txt", // encoded space
|
||||
"..\\u2215..\\u2215etc/passwd", // Unicode division slash
|
||||
"logs/latest -> /etc/passwd", // symlink attacks
|
||||
//TODO: add this test case in advanced routing
|
||||
// "/etc/passwd" // absolute path injection
|
||||
}
|
||||
if !s.windowsTests {
|
||||
// Windows strips trailing dots from path components, making '...' an
|
||||
// invalid directory name (the filesystem rejects MkdirAll with it).
|
||||
objs = append(objs, ".../.../.../target.txt")
|
||||
objs = append(objs, "dir/%20../file.txt") // encoded space
|
||||
// literal backslashes are treated as path separators on Windows
|
||||
objs = append(objs, "..\\u2215..\\u2215etc/passwd") // Unicode division slash
|
||||
// On Windows, '>' is an invalid filename character. The key
|
||||
// 'logs/latest -> /etc/passwd' creates a directory component
|
||||
// 'latest -> ' (containing '>') which MkdirAll rejects.
|
||||
objs = append(objs, "logs/latest -> /etc/passwd") // symlink attacks
|
||||
}
|
||||
|
||||
sort.Strings(objs)
|
||||
|
||||
|
||||
@@ -1073,6 +1073,7 @@ func TestPosix(ts *TestState) {
|
||||
ts.Run(PutObject_overwrite_file_obj_with_nested_obj)
|
||||
ts.Run(PutObject_dir_obj_with_data)
|
||||
ts.Run(PutObject_with_slashes)
|
||||
ts.Run(PutObject_windows_reserved_names)
|
||||
ts.Run(CreateMultipartUpload_dir_obj)
|
||||
ts.Run(PutObject_name_too_long)
|
||||
ts.Run(HeadObject_name_too_long)
|
||||
@@ -1185,6 +1186,7 @@ func TestScoutfs(ts *TestState) {
|
||||
ts.Run(PutObject_overwrite_file_obj_with_nested_obj)
|
||||
ts.Run(PutObject_dir_obj_with_data)
|
||||
ts.Run(PutObject_with_slashes)
|
||||
ts.Run(PutObject_windows_reserved_names)
|
||||
ts.Run(CreateMultipartUpload_dir_obj)
|
||||
ts.Run(PutObject_name_too_long)
|
||||
ts.Run(HeadObject_name_too_long)
|
||||
@@ -2084,9 +2086,7 @@ func TestVersioning(ts *TestState) {
|
||||
ts.Run(Versioning_CopyObject_to_itself)
|
||||
ts.Run(Versioning_CopyObject_to_itself_from_the_current_version)
|
||||
ts.Run(Versioning_CopyObject_conditional_writes)
|
||||
if !ts.conf.windowsTests {
|
||||
ts.Run(Versioning_CopyObject_special_chars)
|
||||
}
|
||||
ts.Run(Versioning_CopyObject_special_chars)
|
||||
// HeadObject action
|
||||
ts.Run(Versioning_HeadObject_invalid_versionId)
|
||||
if !ts.conf.windowsTests {
|
||||
@@ -3645,6 +3645,7 @@ func GetIntTests() IntTests {
|
||||
"PutObject_overwrite_file_obj_with_nested_obj": PutObject_overwrite_file_obj_with_nested_obj,
|
||||
"PutObject_dir_obj_with_data": PutObject_dir_obj_with_data,
|
||||
"PutObject_with_slashes": PutObject_with_slashes,
|
||||
"PutObject_windows_reserved_names": PutObject_windows_reserved_names,
|
||||
"PutObject_race_with_delete": PutObject_race_with_delete,
|
||||
"CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj,
|
||||
"PutObject_sse_c_not_implemented": PutObject_sse_c_not_implemented,
|
||||
|
||||
@@ -17,6 +17,10 @@ package integration
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
@@ -114,6 +118,102 @@ func PutObject_with_slashes(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
func PutObject_windows_reserved_names(s *S3Conf) error {
|
||||
testName := "PutObject_windows_reserved_names"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
// Windows can't name a file after these keys: each one is next to
|
||||
// the key Windows would store it as
|
||||
keys := []string{
|
||||
"question?", "question", "asterisk*", `quote"`, "colon:name", "C:drive",
|
||||
"lt<gt>", "pipe|", "tab\tchar",
|
||||
`back\slash`, "back/slash", `..\..\escape`,
|
||||
"trailingdot.", "trailingdot", "trailingspace ", "trailingspace",
|
||||
"dotdir./file", "dotdir/file", "spacedir /file", "spacedir/file",
|
||||
"CON", "nul.txt", "aux/obj", "com1", "LPT9.log", "CONIN$",
|
||||
"pua\uf03f", "pua?",
|
||||
}
|
||||
for _, key := range keys {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
_, err := s3client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
Body: strings.NewReader(key),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("put %q: %w", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, key := range keys {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.GetObject(ctx, &s3.GetObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &key,
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("get %q: %w", key, err)
|
||||
}
|
||||
body, err := io.ReadAll(out.Body)
|
||||
out.Body.Close()
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("read %q: %w", key, err)
|
||||
}
|
||||
if string(body) != key {
|
||||
return fmt.Errorf("expected the body of %q to be the key, instead got %q", key, body)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(keys)
|
||||
var wantContents []string
|
||||
var wantPrefixes []string
|
||||
for _, key := range keys {
|
||||
prefix, _, found := strings.Cut(key, "/")
|
||||
if !found {
|
||||
wantContents = append(wantContents, key)
|
||||
} else if !slices.Contains(wantPrefixes, prefix+"/") {
|
||||
wantPrefixes = append(wantPrefixes, prefix+"/")
|
||||
}
|
||||
}
|
||||
|
||||
for _, delimiter := range []string{"", "/"} {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
|
||||
Bucket: &bucket,
|
||||
Delimiter: &delimiter,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var contents, prefixes []string
|
||||
for _, obj := range res.Contents {
|
||||
contents = append(contents, *obj.Key)
|
||||
}
|
||||
for _, cp := range res.CommonPrefixes {
|
||||
prefixes = append(prefixes, *cp.Prefix)
|
||||
}
|
||||
|
||||
want, wantCps := keys, []string(nil)
|
||||
if delimiter != "" {
|
||||
want, wantCps = wantContents, wantPrefixes
|
||||
}
|
||||
if !slices.Equal(contents, want) {
|
||||
return fmt.Errorf("expected the objects listed with delimiter %q to be %q, instead got %q",
|
||||
delimiter, want, contents)
|
||||
}
|
||||
if !slices.Equal(prefixes, wantCps) {
|
||||
return fmt.Errorf("expected the common prefixes listed with delimiter %q to be %q, instead got %q",
|
||||
delimiter, wantCps, prefixes)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func CreateMultipartUpload_dir_obj(s *S3Conf) error {
|
||||
testName := "CreateMultipartUpload_dir_obj"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
|
||||
@@ -857,10 +857,6 @@ func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error {
|
||||
testName := "PresignedAuth_Put_GetObject_with_UTF8_chars"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error {
|
||||
obj := "my-$%^&*;"
|
||||
if s.windowsTests {
|
||||
// '*' is not a valid filename character on Windows
|
||||
obj = "my-$%^&;"
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj})
|
||||
|
||||
Reference in New Issue
Block a user