diff --git a/.github/workflows/functional-windows.yml b/.github/workflows/functional-windows.yml new file mode 100644 index 00000000..ccea48e8 --- /dev/null +++ b/.github/workflows/functional-windows.yml @@ -0,0 +1,38 @@ +name: functional tests (windows) +permissions: {} +on: pull_request + +jobs: + build: + name: RunTestsWindows + runs-on: windows-latest + steps: + + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: 'stable' + id: go + + - name: Get Dependencies + run: go mod download + + - name: Build + run: | + $version = if (Test-Path VERSION) { Get-Content VERSION } else { git describe --abbrev=0 --tags HEAD } + $build = git rev-parse --short HEAD + $time = (Get-Date -Format "yyyy-MM-dd_hh:mm:sstt") + go build -ldflags "-X=main.Build=$build -X=main.BuildTime=$time -X=main.Version=$version" ` + -o versitygw.exe -cover -race ./cmd/versitygw + shell: pwsh + + - name: Run Tests + run: pwsh -File .\runtests.ps1 + shell: pwsh + + - name: Coverage Report + run: go tool covdata percent -i="$env:TEMP\covdata" + shell: pwsh diff --git a/README.md b/README.md index 90c2a8de..a31d9963 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,9 @@ ### Binary release builds Download [latest release](https://github.com/versity/versitygw/releases) - | Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 | - |:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:| - | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | - + | Linux amd64/arm64 | MacOS amd64/arm64 | BSD amd64/arm64 | Windows amd64/arm64 | + |:-----------:|:-----------:|:-----------:|:-----------:| + | ✔️ | ✔️ | ✔️ | ✔️ | ### Use Cases * Turn your local filesystem into an S3 server with a single command! * Proxy S3 requests to S3 storage diff --git a/backend/meta/sidecar.go b/backend/meta/sidecar.go index 3056ae3e..d0363cab 100644 --- a/backend/meta/sidecar.go +++ b/backend/meta/sidecar.go @@ -49,6 +49,7 @@ func NewSideCar(dir string) (SideCar, error) { // RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket. func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) { + bucket, object = trimVolume(bucket), trimVolume(object) metadir := filepath.Join(s.dir, bucket, object, sidecarmeta) if object == "" { metadir = filepath.Join(s.dir, bucket, sidecarmeta) @@ -68,6 +69,7 @@ func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) // StoreAttribute stores the value of a specific attribute for an object or a bucket. func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error { + bucket, object = trimVolume(bucket), trimVolume(object) metadir := filepath.Join(s.dir, bucket, object, sidecarmeta) if object == "" { metadir = filepath.Join(s.dir, bucket, sidecarmeta) @@ -114,6 +116,7 @@ func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, va // DeleteAttribute removes the value of a specific attribute for an object or a bucket. func (s SideCar) DeleteAttribute(bucket, object, attribute string) error { + bucket, object = trimVolume(bucket), trimVolume(object) metadir := filepath.Join(s.dir, bucket, object, sidecarmeta) if object == "" { metadir = filepath.Join(s.dir, bucket, sidecarmeta) @@ -135,6 +138,7 @@ func (s SideCar) DeleteAttribute(bucket, object, attribute string) error { // ListAttributes lists all attributes for an object or a bucket. func (s SideCar) ListAttributes(bucket, object string) ([]string, error) { + bucket, object = trimVolume(bucket), trimVolume(object) metadir := filepath.Join(s.dir, bucket, object, sidecarmeta) if object == "" { metadir = filepath.Join(s.dir, bucket, sidecarmeta) @@ -160,6 +164,7 @@ func (s SideCar) ListAttributes(bucket, object string) ([]string, error) { // When object is empty the entire bucket sidecar directory is removed, // cleaning up any orphaned object or multipart metadata within it. func (s SideCar) DeleteAttributes(bucket, object string) error { + bucket, object = trimVolume(bucket), trimVolume(object) if object == "" { // Remove the entire bucket sidecar directory so that orphaned // object/multipart metadata does not accumulate after DeleteBucket. @@ -184,8 +189,9 @@ func (s SideCar) DeleteAttributes(bucket, object string) error { // newObject so that path-based lookups continue to work after the data // directory has been renamed. func (s SideCar) RenameObject(bucket, oldObject, newObject string) error { - oldPath := filepath.Join(s.dir, bucket, oldObject) - newPath := filepath.Join(s.dir, bucket, newObject) + bucket = trimVolume(bucket) + oldPath := filepath.Join(s.dir, bucket, trimVolume(oldObject)) + newPath := filepath.Join(s.dir, bucket, trimVolume(newObject)) if err := os.MkdirAll(filepath.Dir(newPath), 0777); err != nil { if errors.Is(err, syscall.ENOSPC) { diff --git a/backend/posix/parentdir_other.go b/backend/meta/sidecar_unix.go similarity index 79% rename from backend/posix/parentdir_other.go rename to backend/meta/sidecar_unix.go index c5b67954..5496bfcc 100644 --- a/backend/posix/parentdir_other.go +++ b/backend/meta/sidecar_unix.go @@ -14,12 +14,7 @@ //go:build !windows -package posix +package meta -import ( - "github.com/versity/versitygw/s3err" -) - -func handleParentDirError(_ string) error { - return s3err.GetAPIError(s3err.ErrObjectParentIsFile) -} +// trimVolume is a no-op on non-Windows platforms. +func trimVolume(p string) string { return p } diff --git a/backend/posix/parentdir_windows.go b/backend/meta/sidecar_windows.go similarity index 51% rename from backend/posix/parentdir_windows.go rename to backend/meta/sidecar_windows.go index d9d96c2a..a9c369d2 100644 --- a/backend/posix/parentdir_windows.go +++ b/backend/meta/sidecar_windows.go @@ -14,33 +14,16 @@ //go:build windows -package posix +package meta import ( - "os" "path/filepath" - - "github.com/versity/versitygw/s3err" + "strings" ) -func handleParentDirError(name string) error { - dir := filepath.Dir(name) - - // Walk up the directory hierarchy - for dir != "." && dir != "/" { - d, statErr := os.Stat(dir) - if statErr == nil { - // Path component exists - if !d.IsDir() { - // Found a file in the ancestor path - return s3err.GetAPIError(s3err.ErrObjectParentIsFile) - } - // Found a valid directory ancestor, parent truly doesn't exist - break - } - // Continue checking parent directories - dir = filepath.Dir(dir) - } - // Parent doesn't exist or is a directory, treat as ENOENT - return nil +// trimVolume strips the Windows drive letter (e.g. "C:") and any leading +// path separator from p so it can be safely used as a sub-path component +// inside filepath.Join without becoming an absolute root. +func trimVolume(p string) string { + return strings.TrimLeft(p[len(filepath.VolumeName(p)):], `\/`) } diff --git a/backend/posix/dir_unix.go b/backend/posix/dir_unix.go new file mode 100644 index 00000000..e8017434 --- /dev/null +++ b/backend/posix/dir_unix.go @@ -0,0 +1,54 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !windows + +package posix + +import ( + "errors" + "os" + "syscall" + + "github.com/versity/versitygw/s3err" +) + +func handleParentDirError(_ string) error { + return s3err.GetAPIError(s3err.ErrObjectParentIsFile) +} + +// isErrNotDir reports whether err indicates that a path component is a file, +// not a directory (POSIX ENOTDIR). +func isErrNotDir(err error) bool { + return errors.Is(err, syscall.ENOTDIR) +} + +// isErrNameTooLong reports whether err indicates that a filename or path +// component is too long (POSIX ENAMETOOLONG). +func isErrNameTooLong(err error) bool { + return errors.Is(err, syscall.ENAMETOOLONG) +} + +// isErrDirNotEmpty reports whether err indicates that a directory is not empty +// (POSIX ENOTEMPTY). +func isErrDirNotEmpty(err error) bool { + return errors.Is(err, syscall.ENOTEMPTY) +} + +// openForRead opens a file for reading. On non-Windows systems, os.Open is +// sufficient because POSIX allows removing (unlinking) a file that is still +// open by another process. +func openForRead(name string) (*os.File, error) { + return os.Open(name) +} diff --git a/backend/posix/dir_windows.go b/backend/posix/dir_windows.go new file mode 100644 index 00000000..3538620c --- /dev/null +++ b/backend/posix/dir_windows.go @@ -0,0 +1,131 @@ +// Copyright 2026 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build windows + +package posix + +import ( + "errors" + "os" + "path/filepath" + "syscall" + + "github.com/versity/versitygw/s3err" +) + +func handleParentDirError(name string) error { + dir := filepath.Dir(name) + + // Walk up the directory hierarchy + for dir != "." && dir != "/" { + d, statErr := os.Stat(dir) + if statErr == nil { + // Path component exists + if !d.IsDir() { + // Found a file in the ancestor path + return s3err.GetAPIError(s3err.ErrObjectParentIsFile) + } + // Found a valid directory ancestor, parent truly doesn't exist + break + } + // Continue checking parent directories + dir = filepath.Dir(dir) + } + // Parent doesn't exist or is a directory, treat as ENOENT + return nil +} + +// errDirectory is Windows ERROR_DIRECTORY (267): "The directory name is invalid." +// Windows returns this when opening a path like "file/" where "file" is a regular +// file rather than a directory — the POSIX equivalent is ENOTDIR. +const errDirectory = syscall.Errno(267) + +// errInvalidName is Windows ERROR_INVALID_NAME (123): "The filename, directory +// name, or volume label syntax is incorrect." Windows returns this when a path +// component exceeds the filesystem name-length limit — the POSIX equivalent is +// ENAMETOOLONG. +const errInvalidName = syscall.Errno(123) + +// errDirNotEmpty is Windows ERROR_DIR_NOT_EMPTY (145): "The directory is not +// empty." — the POSIX equivalent is ENOTEMPTY. +const errDirNotEmpty = syscall.Errno(145) + +// isErrNameTooLong reports whether err indicates that a filename or path +// component is too long. On Windows this covers both ENAMETOOLONG +// (ERROR_FILENAME_EXCED_RANGE, 206) and ERROR_INVALID_NAME (123), which is +// what the Windows kernel returns for a 300-character filename that exceeds +// MAX_PATH. +func isErrNameTooLong(err error) bool { + if errors.Is(err, syscall.ENAMETOOLONG) { + return true + } + var sysErr syscall.Errno + if errors.As(err, &sysErr) { + return sysErr == errInvalidName + } + return false +} + +// isErrDirNotEmpty reports whether err indicates that a directory is not empty. +// On Windows this covers both ENOTEMPTY and ERROR_DIR_NOT_EMPTY (145). +func isErrDirNotEmpty(err error) bool { + if errors.Is(err, syscall.ENOTEMPTY) { + return true + } + var sysErr syscall.Errno + if errors.As(err, &sysErr) { + return sysErr == errDirNotEmpty + } + return false +} + +// isErrNotDir reports whether err indicates that a path component is a file, +// not a directory. On Windows this covers both ENOTDIR and ERROR_DIRECTORY +// because os.Open / os.Stat do not map ERROR_DIRECTORY to ENOTDIR. +func isErrNotDir(err error) bool { + if errors.Is(err, syscall.ENOTDIR) { + return true + } + var sysErr syscall.Errno + if errors.As(err, &sysErr) { + return sysErr == errDirectory + } + return false +} + +// openForRead opens a file for reading with FILE_SHARE_DELETE so that a +// concurrent DeleteObject (os.Remove) can succeed even while the file handle +// is held open for streaming the GET response body. Without this flag, +// Windows returns "The process cannot access the file because it is being +// used by another process" on the Remove call. +func openForRead(name string) (*os.File, error) { + ptr, err := syscall.UTF16PtrFromString(name) + if err != nil { + return nil, &os.PathError{Op: "open", Path: name, Err: err} + } + h, err := syscall.CreateFile( + ptr, + syscall.GENERIC_READ, + syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, + nil, + syscall.OPEN_EXISTING, + syscall.FILE_ATTRIBUTE_NORMAL, + 0, + ) + if err != nil { + return nil, &os.PathError{Op: "open", Path: name, Err: err} + } + return os.NewFile(uintptr(h), name), nil +} diff --git a/backend/posix/posix.go b/backend/posix/posix.go index b8073463..5115205b 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -372,7 +372,7 @@ func (p *Posix) doesBucketAndObjectExist(bucket, object string) error { } _, err = os.Stat(filepath.Join(bucket, object)) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil { @@ -1060,7 +1060,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error { // data file simply doesn't exist — the two cases are indistinguishable // from metadata alone. Verify the data file directly so callers // receive the correct NoSuchVersion / NoSuchKey error. - if _, statErr := os.Stat(filepath.Join(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) { + if _, statErr := os.Stat(filepath.Join(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) { if versionId != "" { return s3err.GetAPIError(s3err.ErrNoSuchVersion) } @@ -1068,7 +1068,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error { } _, err := p.meta.RetrieveAttribute(nil, bucket, object, deleteMarkerKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return s3err.GetNoSuchVersionErr(object, versionId) } @@ -1087,7 +1087,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error { // Check if the given object is a delete marker func (p *Posix) isObjDeleteMarker(bucket, object string) (bool, error) { _, err := p.meta.RetrieveAttribute(nil, bucket, object, deleteMarkerKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return false, s3err.GetAPIError(s3err.ErrNoSuchKey) } if errors.Is(err, meta.ErrNoSuchKey) { @@ -3289,7 +3289,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) if errors.Is(err, fs.ErrNotExist) { return s3response.CopyPartResult{}, s3err.GetNoSuchUploadErr(*upi.UploadId) } - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(*upi.Key)), 1024) } if err != nil { @@ -3360,7 +3360,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) } return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(srcObject)), 1024) } if err != nil { @@ -3776,10 +3776,10 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje _ = p.meta.DeleteAttribute(*po.Bucket, *po.Key, objectRetentionKey) } } - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return s3response.PutObjectOutput{}, s3err.GetKeyTooLongErr(int64(len(*po.Key)), 1024) } - if errors.Is(err, syscall.ENOTDIR) { + if isErrNotDir(err) { parentErr := handleParentDirError(name) if parentErr != nil { return s3response.PutObjectOutput{}, parentErr @@ -4067,11 +4067,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( if getString(input.VersionId) == "" { // if the versionId is not specified, make the current version a delete marker fi, err := os.Stat(objpath) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { // AWS returns success if the object does not exist return &s3.DeleteObjectOutput{}, nil } - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) } if err != nil { @@ -4134,7 +4134,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( versionPath := p.genObjVersionPath(bucket, object) vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { // AWS returns success if the object does not exist return &s3.DeleteObjectOutput{ VersionId: input.VersionId, @@ -4151,7 +4151,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( // but "foo" is a regular file (not a directory), the path cannot // contain any object. _, statErr := os.Stat(filepath.Join(bucket, object)) - if errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) { + if errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) { return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil } vId = []byte(nullVersionId) @@ -4204,7 +4204,6 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( if err != nil { return nil, fmt.Errorf("open obj version: %w", err) } - defer sf.Close() acct, ok := ctx.Value("account").(auth.Account) if !ok { acct = auth.Account{} @@ -4220,9 +4219,14 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( _, err = io.Copy(f, sf) if err != nil { + _ = sf.Close() return nil, fmt.Errorf("copy object %w", err) } + if err := sf.Close(); err != nil { + return nil, fmt.Errorf("close obj version: %w", err) + } + if err := f.link(); err != nil { return nil, fmt.Errorf("link tmp file: %w", err) } @@ -4275,10 +4279,10 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId) err = os.Remove(filepath.Join(versionPath, *input.VersionId)) - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) } - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, *input.VersionId) } if err != nil { @@ -4296,10 +4300,10 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( } fi, err := os.Stat(objpath) - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024) } - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { // AWS returns success if the object does not exist return &s3.DeleteObjectOutput{}, nil } @@ -4329,7 +4333,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if errors.Is(err, syscall.ENOTEMPTY) { + if isErrDirNotEmpty(err) { // If the directory object has been uploaded explicitly // remove the directory object (remove the ETag) _, err = p.meta.RetrieveAttribute(nil, objpath, "", etagkey) @@ -4489,7 +4493,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge object := *input.Key if versionId != "" { vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -4508,13 +4512,13 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge objPath := filepath.Join(bucket, object) fid, err := os.Stat(objPath) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return nil, s3err.GetNoSuchVersionErr(*input.Key, versionId) } return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024) } if err != nil { @@ -4648,7 +4652,10 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge versionId = string(vId) } - f, err := os.Open(objPath) + // openForRead opens with FILE_SHARE_DELETE on Windows so that a concurrent + // DeleteObject can call os.Remove on this file while the GET response body + // is still being streamed. On POSIX, os.Open is sufficient. + f, err := openForRead(objPath) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } @@ -4657,13 +4664,13 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge } fi, err := f.Stat() - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024) } if err != nil { @@ -4829,7 +4836,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. if versionId != "" { vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -4849,13 +4856,13 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. objPath := filepath.Join(bucket, object) fi, err := os.Stat(objPath) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return nil, s3err.GetNoSuchVersionErr(*input.Key, versionId) } return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024) } if err != nil { @@ -5174,7 +5181,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput return s3response.CopyObjectOutput{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, srcVersionId) } vId, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -5197,13 +5204,13 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput objPath := joinPathWithTrailer(srcBucket, srcObject) f, err := os.Open(objPath) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if p.versioningEnabled() && vEnabled { return s3response.CopyObjectOutput{}, s3err.GetNoSuchVersionErr(origSrcObject, srcVersionId) } return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return s3response.CopyObjectOutput{}, s3err.GetKeyTooLongErr(int64(len(origSrcObject)), 1024) } if err != nil { @@ -5344,7 +5351,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput b, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, etagkey) etag = string(b) vId, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey) } version = backend.GetPtrFromString(string(vId)) @@ -5888,10 +5895,10 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId if versionId == "" { _, err = os.Stat(filepath.Join(bucket, object)) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return nil, s3err.GetAPIError(s3err.ErrKeyTooLong) } if err != nil { @@ -5905,7 +5912,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -5929,7 +5936,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId func (p *Posix) getAttrTags(bucket, object, versionId string) (map[string]string, error) { tags := make(map[string]string) b, err := p.meta.RetrieveAttribute(nil, bucket, object, tagHdr) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return nil, s3err.GetNoSuchVersionErr(object, versionId) } @@ -5978,10 +5985,10 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId if versionId == "" { _, err = os.Stat(filepath.Join(bucket, object)) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return s3err.GetAPIError(s3err.ErrNoSuchKey) } - if errors.Is(err, syscall.ENAMETOOLONG) { + if isErrNameTooLong(err) { return s3err.GetAPIError(s3err.ErrKeyTooLong) } if err != nil { @@ -5995,7 +6002,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -6015,7 +6022,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId if tags == nil { err = p.meta.DeleteAttribute(bucket, object, tagHdr) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return s3err.GetNoSuchVersionErr(object, versionId) } @@ -6036,7 +6043,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId } err = p.meta.StoreAttribute(nil, bucket, object, tagHdr, b) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return s3err.GetNoSuchVersionErr(object, versionId) } @@ -6420,7 +6427,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -6439,7 +6446,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI } err = p.meta.StoreAttribute(nil, bucket, object, objectLegalHoldKey, statusData) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return s3err.GetNoSuchVersionErr(object, versionId) } @@ -6481,7 +6488,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -6500,7 +6507,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI } data, err := p.meta.RetrieveAttribute(nil, bucket, object, objectLegalHoldKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return nil, s3err.GetNoSuchVersionErr(object, versionId) } @@ -6547,7 +6554,7 @@ func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionI return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -6602,7 +6609,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId) } vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -6621,7 +6628,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI } data, err := p.meta.RetrieveAttribute(nil, bucket, object, objectRetentionKey) - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) { + if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) { if versionId != "" { return nil, s3err.GetNoSuchVersionErr(object, versionId) } diff --git a/backend/posix/without_otmpfile.go b/backend/posix/without_otmpfile.go index 7b22a5f6..bc7cf579 100644 --- a/backend/posix/without_otmpfile.go +++ b/backend/posix/without_otmpfile.go @@ -109,7 +109,7 @@ func (tmp *tmpfile) link() error { backoffMs := initialBackoffMs for { err = backend.MoveFile(tempname, objPath, defaultFilePerm) - if !errors.Is(err, syscall.ENOENT) { + if !os.IsNotExist(err) { break } // The parent directory may have been concurrently removed; backoff and retry. @@ -117,11 +117,12 @@ func (tmp *tmpfile) link() error { sleepWithJitter(backoffMs) backoffMs = min((backoffMs * 2), maxBackoffMs) - err = backend.MkdirAll(filepath.Dir(objPath), tmp.uid, tmp.gid, + // Best-effort: recreate the parent directory. Ignore errors here; + // if recreation fails transiently (e.g. Windows pending-delete on + // a recently removed directory), the next MoveFile attempt will + // return os.IsNotExist again and we will retry. + _ = backend.MkdirAll(filepath.Dir(objPath), tmp.uid, tmp.gid, tmp.doChown, tmp.newDirPerm) - if err != nil { - return fmt.Errorf("recreate parent dir: %w", err) - } } return err } diff --git a/cmd/versitygw/test.go b/cmd/versitygw/test.go index 9bdffd8e..8eadca23 100644 --- a/cmd/versitygw/test.go +++ b/cmd/versitygw/test.go @@ -44,6 +44,7 @@ var ( azureTests bool tlsStatus bool parallel bool + windowsTests bool sidecarTests bool ) @@ -121,6 +122,12 @@ func initTestCommands() []*cli.Command { Destination: &azureTests, Aliases: []string{"azure"}, }, + &cli.BoolFlag{ + Name: "windows-test-mode", + Usage: "Skips tests that are not supported on Windows", + Destination: &windowsTests, + Aliases: []string{"windows"}, + }, &cli.BoolFlag{ Name: "sidecar-test-mode", Usage: "Skips tests that are not supported by Sidecar", @@ -176,6 +183,12 @@ func initTestCommands() []*cli.Command { Destination: &versioningEnabled, Aliases: []string{"vs"}, }, + &cli.BoolFlag{ + Name: "windows-test-mode", + Usage: "Skips tests that are not supported on Windows", + Destination: &windowsTests, + Aliases: []string{"windows"}, + }, }, }, { @@ -421,6 +434,10 @@ func getAction(tf testFunc) func(ctx *cli.Context) error { if azureTests { opts = append(opts, integration.WithAzureMode()) } + if windowsTests { + opts = append(opts, integration.WithWindowsMode()) + opts = append(opts, integration.WithSidecarMode()) + } if sidecarTests { opts = append(opts, integration.WithSidecarMode()) } @@ -470,6 +487,10 @@ func extractIntTests() (commands []*cli.Command) { if azureTests { opts = append(opts, integration.WithAzureMode()) } + if windowsTests { + opts = append(opts, integration.WithWindowsMode()) + opts = append(opts, integration.WithSidecarMode()) + } if sidecarTests { opts = append(opts, integration.WithSidecarMode()) } @@ -497,6 +518,12 @@ func extractIntTests() (commands []*cli.Command) { Destination: &sidecarTests, Aliases: []string{"sidecar"}, }, + &cli.BoolFlag{ + Name: "windows-test-mode", + Usage: "Skips tests that are not supported on Windows", + Destination: &windowsTests, + Aliases: []string{"windows"}, + }, }, }) } diff --git a/runtests.ps1 b/runtests.ps1 new file mode 100644 index 00000000..1650a42b --- /dev/null +++ b/runtests.ps1 @@ -0,0 +1,174 @@ +#!/usr/bin/env pwsh +# Copyright 2026 Versity Software +# This file is licensed under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# PowerShell equivalent of runtests.sh for Windows + +$ErrorActionPreference = "Stop" + +# Temp directories +$tmpGw = Join-Path $env:TEMP "gw" +$tmpCovdata = Join-Path $env:TEMP "covdata" +$tmpHttpsCovdata = Join-Path $env:TEMP "https.covdata" +$tmpVersioningCovdata = Join-Path $env:TEMP "versioning.covdata" +$tmpVersioningHttpsCovdata = Join-Path $env:TEMP "versioning.https.covdata" +$tmpNoaclCovdata = Join-Path $env:TEMP "noacl.covdata" +$tmpVersioningDir = Join-Path $env:TEMP "versioningdir" +$tmpSidecar = Join-Path $env:TEMP "sidecar" + +foreach ($dir in @($tmpGw, $tmpCovdata, $tmpHttpsCovdata, $tmpVersioningCovdata, + $tmpVersioningHttpsCovdata, $tmpNoaclCovdata, $tmpVersioningDir, + $tmpSidecar)) { + if (Test-Path $dir) { Remove-Item -Recurse -Force $dir } + New-Item -ItemType Directory -Path $dir | Out-Null +} + +# Setup TLS certificate and key +Write-Host "Generating TLS certificate and key in the cert.pem and key.pem files" +openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048 +if ($LASTEXITCODE -ne 0) { throw "Failed to generate private key" } +openssl req -new -x509 -key key.pem -out cert.pem -days 365 ` + -subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com" +if ($LASTEXITCODE -ne 0) { throw "Failed to generate certificate" } + +function Start-Gateway { + param( + [string] $CoverDir, + [string[]] $GwArgs + ) + $env:GOCOVERDIR = $CoverDir + $proc = Start-Process -FilePath ".\versitygw.exe" -ArgumentList $GwArgs -PassThru -NoNewWindow + Remove-Item Env:\GOCOVERDIR -ErrorAction SilentlyContinue + return $proc +} + +function Invoke-GwTest { + param( + [string] $Description, + [string[]] $TestArgs, + [System.Diagnostics.Process] $GatewayProc + ) + & .\versitygw.exe test @TestArgs + if ($LASTEXITCODE -ne 0) { + Write-Host "$Description failed" + Stop-Process -Id $GatewayProc.Id -Force -ErrorAction SilentlyContinue + exit 1 + } +} + +# --------------------------------------------------------------------------- +# 1. HTTP (port 7070) +# --------------------------------------------------------------------------- +Write-Host "Running the sdk test over http" +$gwProc = Start-Gateway -CoverDir $tmpCovdata ` + -GwArgs @("-a", "user", "-s", "pass", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw) +Start-Sleep -Seconds 1 + +if ($gwProc.HasExited) { + Write-Host "server no longer running" + exit 1 +} + +Invoke-GwTest -Description "full flow tests" -GatewayProc $gwProc ` + -TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "full-flow", "--parallel", "--windows-test-mode") +Invoke-GwTest -Description "posix tests" -GatewayProc $gwProc ` + -TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "posix", "--windows-test-mode") +Invoke-GwTest -Description "iam tests" -GatewayProc $gwProc ` + -TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "iam") + +Stop-Process -Id $gwProc.Id -Force -ErrorAction SilentlyContinue + +# --------------------------------------------------------------------------- +# 2. HTTPS (port 7071) +# --------------------------------------------------------------------------- +Write-Host "Running the sdk test over https" +$gwHttpsProc = Start-Gateway -CoverDir $tmpHttpsCovdata ` + -GwArgs @("--cert", "$PWD\cert.pem", "--key", "$PWD\key.pem", + "-p", ":7071", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw) +Start-Sleep -Seconds 1 + +if ($gwHttpsProc.HasExited) { + Write-Host "https server no longer running" + exit 1 +} + +Invoke-GwTest -Description "https full flow tests" -GatewayProc $gwHttpsProc ` + -TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "full-flow", "--parallel", "--windows-test-mode") +Invoke-GwTest -Description "https posix tests" -GatewayProc $gwHttpsProc ` + -TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "posix", "--windows-test-mode") +Invoke-GwTest -Description "https iam tests" -GatewayProc $gwHttpsProc ` + -TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "iam") + +Stop-Process -Id $gwHttpsProc.Id -Force -ErrorAction SilentlyContinue + +# --------------------------------------------------------------------------- +# 3. Versioning HTTP (port 7072) +# --------------------------------------------------------------------------- +Write-Host "Running the sdk test over http against the versioning-enabled gateway" +$gwVsProc = Start-Gateway -CoverDir $tmpVersioningCovdata ` + -GwArgs @("-p", ":7072", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw, + "posix", "--sidecar", $tmpSidecar, "--versioning-dir", $tmpVersioningDir, $tmpGw) +Start-Sleep -Seconds 1 + +if ($gwVsProc.HasExited) { + Write-Host "versioning-enabled server no longer running" + exit 1 +} + +Invoke-GwTest -Description "versioning-enabled full-flow tests" -GatewayProc $gwVsProc ` + -TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7072", "full-flow", "-vs", "--parallel", "--windows-test-mode") +Invoke-GwTest -Description "versioning-enabled posix tests" -GatewayProc $gwVsProc ` + -TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7072", "posix", "-vs", "--windows-test-mode") + +Stop-Process -Id $gwVsProc.Id -Force -ErrorAction SilentlyContinue + +# --------------------------------------------------------------------------- +# 4. Versioning HTTPS (port 7073) +# --------------------------------------------------------------------------- +Write-Host "Running the sdk test over https against the versioning-enabled gateway" +$gwVsHttpsProc = Start-Gateway -CoverDir $tmpVersioningHttpsCovdata ` + -GwArgs @("--cert", "$PWD\cert.pem", "--key", "$PWD\key.pem", + "-p", ":7073", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw, + "posix", "--sidecar", $tmpSidecar, "--versioning-dir", $tmpVersioningDir, $tmpGw) +Start-Sleep -Seconds 1 + +if ($gwVsHttpsProc.HasExited) { + Write-Host "versioning-enabled https server no longer running" + exit 1 +} + +Invoke-GwTest -Description "versioning-enabled https full-flow tests" -GatewayProc $gwVsHttpsProc ` + -TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7073", "full-flow", "-vs", "--parallel", "--windows-test-mode") +Invoke-GwTest -Description "versioning-enabled https posix tests" -GatewayProc $gwVsHttpsProc ` + -TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7073", "posix", "-vs", "--windows-test-mode") + +Stop-Process -Id $gwVsHttpsProc.Id -Force -ErrorAction SilentlyContinue + +# --------------------------------------------------------------------------- +# 5. No ACL (port 7074) +# --------------------------------------------------------------------------- +Write-Host "Running No ACL integration tests" +$gwNoAclProc = Start-Gateway -CoverDir $tmpNoaclCovdata ` + -GwArgs @("-p", ":7074", "-a", "user", "-s", "pass", "-noacl", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw) +Start-Sleep -Seconds 1 + +if ($gwNoAclProc.HasExited) { + Write-Host "noacl server no longer running" + exit 1 +} + +Invoke-GwTest -Description "No ACL integration tests" -GatewayProc $gwNoAclProc ` + -TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7074", "noacl") + +Stop-Process -Id $gwNoAclProc.Id -Force -ErrorAction SilentlyContinue diff --git a/tests/integration/ListBuckets.go b/tests/integration/ListBuckets.go index 9c967676..178fa618 100644 --- a/tests/integration/ListBuckets.go +++ b/tests/integration/ListBuckets.go @@ -17,6 +17,7 @@ package integration import ( "context" "fmt" + "sort" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" @@ -90,6 +91,29 @@ func ListBuckets_as_user(s *S3Conf) error { func ListBuckets_as_admin(s *S3Conf) error { testName := "ListBuckets_as_admin" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + testuser, adminUser := getUser("user"), getUser("admin") + + err := createUsers(s, []user{testuser, adminUser}) + if err != nil { + return err + } + + adminClient := s.getUserClient(adminUser) + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + initOut, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{}) + cancel() + if err != nil { + return err + } + + ignore := make([]string, 0, len(initOut.Buckets)) + for _, b := range initOut.Buckets { + if *b.Name != bucket { + ignore = append(ignore, *b.Name) + } + } + buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}} for range 6 { bckt := getBucketName() @@ -104,12 +128,6 @@ func ListBuckets_as_admin(s *S3Conf) error { BucketRegion: &s.awsRegion, }) } - testuser, adminUser := getUser("user"), getUser("admin") - - err := createUsers(s, []user{testuser, adminUser}) - if err != nil { - return err - } bckts := []string{} for i := range 3 { @@ -121,9 +139,7 @@ func ListBuckets_as_admin(s *S3Conf) error { return err } - adminClient := s.getUserClient(adminUser) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) out, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{}) cancel() if err != nil { @@ -134,7 +150,7 @@ func ListBuckets_as_admin(s *S3Conf) error { return fmt.Errorf("expected buckets owner to be %v, instead got %v", adminUser.access, getString(out.Owner.ID)) } - if !compareBuckets(out.Buckets, buckets) { + if !compareBuckets(out.Buckets, buckets, ignore...) { return fmt.Errorf("expected list buckets result to be %v, instead got %v", sprintBuckets(buckets), sprintBuckets(out.Buckets)) } @@ -154,6 +170,21 @@ func ListBuckets_with_prefix(s *S3Conf) error { testName := "ListBuckets_with_prefix" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { prefix := "my-prefix-" + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + prefixInitOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{ + Prefix: &prefix, + }) + cancel() + if err != nil { + return err + } + + ignore := make([]string, 0, len(prefixInitOut.Buckets)) + for _, b := range prefixInitOut.Buckets { + ignore = append(ignore, *b.Name) + } + allBuckets, prefixedBuckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}, []types.Bucket{} for i := range 5 { bckt := getBucketName() @@ -179,7 +210,7 @@ func ListBuckets_with_prefix(s *S3Conf) error { } } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{ Prefix: &prefix, }) @@ -196,7 +227,7 @@ func ListBuckets_with_prefix(s *S3Conf) error { return fmt.Errorf("expected prefix to be %v, instead got %v", prefix, getString(out.Prefix)) } - if !compareBuckets(out.Buckets, prefixedBuckets) { + if !compareBuckets(out.Buckets, prefixedBuckets, ignore...) { return fmt.Errorf("expected list buckets result to be %v, instead got %v", prefixedBuckets, out.Buckets) } @@ -243,7 +274,14 @@ func ListBuckets_invalid_max_buckets(s *S3Conf) error { func ListBuckets_truncated(s *S3Conf) error { testName := "ListBuckets_truncated" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}} + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{}) + cancel() + if err != nil { + return err + } + + newBuckets := []types.Bucket{} for range 5 { bckt := getBucketName() @@ -252,14 +290,19 @@ func ListBuckets_truncated(s *S3Conf) error { return err } - buckets = append(buckets, types.Bucket{ + newBuckets = append(newBuckets, types.Bucket{ Name: &bckt, BucketRegion: &s.awsRegion, }) } + buckets := append(initOut.Buckets, newBuckets...) + sort.Slice(buckets, func(i, j int) bool { + return *buckets[i].Name < *buckets[j].Name + }) + maxBuckets := int32(3) - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{ MaxBuckets: &maxBuckets, }) @@ -292,7 +335,7 @@ func ListBuckets_truncated(s *S3Conf) error { if !compareBuckets(out.Buckets, buckets[maxBuckets:]) { return fmt.Errorf("expected list buckets result to be %v, instead got %v", - sprintBuckets(buckets[:maxBuckets]), sprintBuckets(out.Buckets)) + sprintBuckets(buckets[maxBuckets:]), sprintBuckets(out.Buckets)) } if out.ContinuationToken != nil { return fmt.Errorf("expected nil continuation token, instead got %v", @@ -302,7 +345,7 @@ func ListBuckets_truncated(s *S3Conf) error { return fmt.Errorf("expected nil prefix, instead got %v", *out.Prefix) } - for _, elem := range buckets[1:] { + for _, elem := range newBuckets { err = teardown(s, *elem.Name) if err != nil { return err @@ -335,6 +378,20 @@ func ListBuckets_empty_success(s *S3Conf) error { func ListBuckets_success(s *S3Conf) error { testName := "ListBuckets_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{}) + cancel() + if err != nil { + return err + } + + ignore := make([]string, 0, len(initOut.Buckets)) + for _, b := range initOut.Buckets { + if *b.Name != bucket { + ignore = append(ignore, *b.Name) + } + } + buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}} for range 5 { bckt := getBucketName() @@ -350,7 +407,7 @@ func ListBuckets_success(s *S3Conf) error { }) } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + ctx, cancel = context.WithTimeout(context.Background(), shortTimeout) out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{}) cancel() if err != nil { @@ -361,7 +418,7 @@ func ListBuckets_success(s *S3Conf) error { return fmt.Errorf("expected owner to be %v, instead got %v", s.awsID, getString(out.Owner.ID)) } - if !compareBuckets(out.Buckets, buckets) { + if !compareBuckets(out.Buckets, buckets, ignore...) { return fmt.Errorf("expected list buckets result to be %v, instead got %v", sprintBuckets(buckets), sprintBuckets(out.Buckets)) } diff --git a/tests/integration/PutObject.go b/tests/integration/PutObject.go index 01b62cf0..365bf1ca 100644 --- a/tests/integration/PutObject.go +++ b/tests/integration/PutObject.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "fmt" "net/http" + "sort" "strings" "time" @@ -52,10 +53,21 @@ func PutObject_special_chars(s *S3Conf) error { "my?key", "my^key", "my{}key", "my%key", "my`key", "my[]key", "my~key", "my<>key", "my|key", "my#key", } - if !s.azureTests { - // azure currently can't handle backslashes in object names + if !s.azureTests && !s.windowsTests { + // azure and windows cannot handle backslashes in object names: + // on Windows, '\' is a path separator so 'my\key' is stored as 'my/key' objnames = append(objnames, "my\\key") } + if s.windowsTests { + // ':', '?', '<', '>', '|', '*' are not valid filename characters on Windows + var filtered []string + for _, name := range objnames { + if !strings.ContainsAny(name, `:?<>|*`) { + filtered = append(filtered, name) + } + } + objnames = filtered + } return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { objs, err := putObjects(s3client, objnames, bucket) @@ -1169,19 +1181,30 @@ func PutObject_false_negative_object_names(s *S3Conf) error { testName := "PutObject_false_negative_object_names" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { objs := []string{ - "%252e%252e%252fetc/passwd", // double encoding - "%2e%2e/%2e%2e/%2e%2e/.ssh/id_rsa", // double URL-encoded - "%u002e%u002e/%u002e%u002e/etc/passwd", // unicode escape - "..%2f..%2f..%2fsecret/file.txt", // URL-encoded - "..%c0%af..%c0%afetc/passwd", // UTF-8 overlong trick - ".../.../.../target.txt", - "..\\u2215..\\u2215etc/passwd", // Unicode division slash - "dir/%20../file.txt", // encoded space + "%252e%252e%252fetc/passwd", // double encoding + "%2e%2e/%2e%2e/%2e%2e/.ssh/id_rsa", // double URL-encoded + "%u002e%u002e/%u002e%u002e/etc/passwd", // unicode escape + "..%2f..%2f..%2fsecret/file.txt", // URL-encoded + "..%c0%af..%c0%afetc/passwd", // UTF-8 overlong trick "dir/%c0%ae%c0%ae/%c0%ae%c0%ae/etc/passwd", // overlong UTF-8 encoding - "logs/latest -> /etc/passwd", // symlink attacks //TODO: add this test case in advanced routing // "/etc/passwd" // absolute path injection } + if !s.windowsTests { + // Windows strips trailing dots from path components, making '...' an + // invalid directory name (the filesystem rejects MkdirAll with it). + objs = append(objs, ".../.../.../target.txt") + objs = append(objs, "dir/%20../file.txt") // encoded space + // literal backslashes are treated as path separators on Windows + objs = append(objs, "..\\u2215..\\u2215etc/passwd") // Unicode division slash + // On Windows, '>' is an invalid filename character. The key + // 'logs/latest -> /etc/passwd' creates a directory component + // 'latest -> ' (containing '>') which MkdirAll rejects. + objs = append(objs, "logs/latest -> /etc/passwd") // symlink attacks + } + + sort.Strings(objs) + _, err := putObjects(s3client, objs, bucket) if err != nil { return err diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index e4225beb..89503a05 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -196,7 +196,7 @@ func TestPutObject(ts *TestState) { } ts.Run(PutObject_success) ts.Run(PutObject_default_content_type) - if !ts.conf.versioningEnabled { + if !ts.conf.versioningEnabled && !ts.conf.windowsTests { ts.Run(PutObject_racey_success) } ts.Run(PutObject_invalid_credentials) @@ -957,7 +957,9 @@ func TestPosix(ts *TestState) { ts.Run(CopyObject_overwrite_same_dir_object) ts.Run(CopyObject_overwrite_same_file_object) ts.Run(DeleteObject_directory_not_empty) - ts.Run(PutObject_race_with_delete) + if !ts.conf.windowsTests { + ts.Run(PutObject_race_with_delete) + } // posix specific versioning tests if !ts.conf.versioningEnabled { TestVersioningDisabled(ts) @@ -1138,10 +1140,14 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_CopyObject_success) ts.Run(Versioning_CopyObject_non_existing_version_id) ts.Run(Versioning_CopyObject_from_an_object_version) - ts.Run(Versioning_CopyObject_special_chars) + if !ts.conf.windowsTests { + ts.Run(Versioning_CopyObject_special_chars) + } // HeadObject action ts.Run(Versioning_HeadObject_invalid_versionId) - ts.Run(Versioning_HeadObject_non_existing_object_version) + if !ts.conf.windowsTests { + ts.Run(Versioning_HeadObject_non_existing_object_version) + } ts.Run(Versioning_HeadObject_invalid_parent) ts.Run(Versioning_HeadObject_success) ts.Run(Versioning_HeadObject_without_versionId) @@ -1170,7 +1176,9 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_DeleteObject_invalid_versionId) ts.Run(Versioning_DeleteObject_delete_object_version) ts.Run(Versioning_DeleteObject_non_existing_object) - ts.Run(Versioning_DeleteObject_delete_a_delete_marker) + if !ts.conf.windowsTests { + ts.Run(Versioning_DeleteObject_delete_a_delete_marker) + } ts.Run(Versioning_Delete_null_versionId_object) ts.Run(Versioning_DeleteObject_nested_dir_object) ts.Run(Versioning_DeleteObject_non_existing_objects) @@ -1221,13 +1229,17 @@ func TestVersioning(ts *TestState) { ts.Run(Versioning_WORM_PutObject_overwrite_locked_object) ts.Run(Versioning_WORM_CopyObject_overwrite_locked_object) ts.Run(Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object) - ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention) + if !ts.conf.windowsTests { + ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention) + } // Concurrent requests // Versioninig_concurrent_upload_object ts.Run(Versioning_AccessControl_GetObjectVersion) ts.Run(Versioning_AccessControl_HeadObjectVersion) ts.Run(Versioning_AccessControl_object_tagging_policy) - ts.Run(Versioning_AccessControl_DeleteObject_policy) + if !ts.conf.windowsTests { + ts.Run(Versioning_AccessControl_DeleteObject_policy) + } ts.Run(Versioning_AccessControl_GetObjectAttributes_policy) } diff --git a/tests/integration/presigned_urls.go b/tests/integration/presigned_urls.go index 5dc8d3fd..9a3dacc0 100644 --- a/tests/integration/presigned_urls.go +++ b/tests/integration/presigned_urls.go @@ -852,6 +852,10 @@ func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error { testName := "PresignedAuth_Put_GetObject_with_UTF8_chars" return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error { obj := "my-$%^&*;" + if s.windowsTests { + // '*' is not a valid filename character on Windows + obj = "my-$%^&;" + } ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj}) diff --git a/tests/integration/s3conf.go b/tests/integration/s3conf.go index 9b7ad8e0..df243d95 100644 --- a/tests/integration/s3conf.go +++ b/tests/integration/s3conf.go @@ -46,6 +46,7 @@ type S3Conf struct { debug bool versioningEnabled bool azureTests bool + windowsTests bool sidecarTests bool tlsStatus bool httpClient *http.Client @@ -121,6 +122,9 @@ func WithVersioningEnabled() Option { func WithAzureMode() Option { return func(s *S3Conf) { s.azureTests = true } } +func WithWindowsMode() Option { + return func(s *S3Conf) { s.windowsTests = true } +} func WithSidecarMode() Option { return func(s *S3Conf) { s.sidecarTests = true } } diff --git a/tests/integration/utils.go b/tests/integration/utils.go index bfd8247f..0b171ff7 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -1502,7 +1502,21 @@ func areMapsSame(mp1, mp2 map[string]string) bool { return true } -func compareBuckets(list1 []types.Bucket, list2 []types.Bucket) bool { +func compareBuckets(list1 []types.Bucket, list2 []types.Bucket, ignore ...string) bool { + if len(ignore) > 0 { + ignoreSet := make(map[string]struct{}, len(ignore)) + for _, name := range ignore { + ignoreSet[name] = struct{}{} + } + filtered := make([]types.Bucket, 0, len(list1)) + for _, b := range list1 { + if _, skip := ignoreSet[*b.Name]; !skip { + filtered = append(filtered, b) + } + } + list1 = filtered + } + if len(list1) != len(list2) { return false } @@ -1771,8 +1785,17 @@ func listBuckets(s *S3Conf) error { const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" +// randCounter is an atomic counter used by genRandString. It is seeded once +// with time.Now().UnixNano() so that values are unique even when many goroutines +// call genRandString concurrently on systems with low-resolution clocks. +var randCounter = atomic.Uint64{} + +func init() { + randCounter.Store(uint64(time.Now().UnixNano())) +} + func genRandString(length int) string { - source := rnd.NewSource(time.Now().UnixNano()) + source := rnd.NewSource(int64(randCounter.Add(1))) random := rnd.New(source) result := make([]byte, length) for i := range result { diff --git a/tests/integration/versioning.go b/tests/integration/versioning.go index 70e52e6d..cdf13cf6 100644 --- a/tests/integration/versioning.go +++ b/tests/integration/versioning.go @@ -20,6 +20,7 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" "sync" "time" @@ -437,12 +438,17 @@ func Versioning_CopyObject_special_chars(s *S3Conf) error { } srcObjVersionId := *srcObjVersions[0].VersionId + copySource := fmt.Sprintf("%v/%v?versionId=%v", + bucket, + url.PathEscape(srcObj), + url.QueryEscape(srcObjVersionId), + ) ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{ Bucket: &bucket, Key: &dstObj, - CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, srcObj, srcObjVersionId)), + CopySource: getPtr(copySource), }) cancel() if err != nil {