feat: add windows functional test coverage and fix some windows behavior

This change adds Windows functional test execution in CI and updates
backend handling so windows filesystem error/path semantics map correctly
to expected S3 outcomes.

The only meta supported on windows right now is sidecar, so the tests
in windows mode also skip sidecar skips.

Future work is to address the skips and/or more clearly document
the unsupported/incompatible behavior on windows.

The windows support will still remain best effort, but these tests
should at least flag when future changes introduce incompatible
behavior on windows.
This commit is contained in:
Ben McClelland
2026-04-15 14:55:30 -07:00
parent 7bde76e982
commit 27f04ad5ea
18 changed files with 674 additions and 130 deletions
+38
View File
@@ -0,0 +1,38 @@
name: functional tests (windows)
permissions: {}
on: pull_request
jobs:
build:
name: RunTestsWindows
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: 'stable'
id: go
- name: Get Dependencies
run: go mod download
- name: Build
run: |
$version = if (Test-Path VERSION) { Get-Content VERSION } else { git describe --abbrev=0 --tags HEAD }
$build = git rev-parse --short HEAD
$time = (Get-Date -Format "yyyy-MM-dd_hh:mm:sstt")
go build -ldflags "-X=main.Build=$build -X=main.BuildTime=$time -X=main.Version=$version" `
-o versitygw.exe -cover -race ./cmd/versitygw
shell: pwsh
- name: Run Tests
run: pwsh -File .\runtests.ps1
shell: pwsh
- name: Coverage Report
run: go tool covdata percent -i="$env:TEMP\covdata"
shell: pwsh
+3 -4
View File
@@ -10,10 +10,9 @@
### Binary release builds
Download [latest release](https://github.com/versity/versitygw/releases)
| Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
|:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
| ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
| Linux amd64/arm64 | MacOS amd64/arm64 | BSD amd64/arm64 | Windows amd64/arm64 |
|:-----------:|:-----------:|:-----------:|:-----------:|
| ✔️ | ✔️ | ✔️ | ✔️ |
### Use Cases
* Turn your local filesystem into an S3 server with a single command!
* Proxy S3 requests to S3 storage
+8 -2
View File
@@ -49,6 +49,7 @@ func NewSideCar(dir string) (SideCar, error) {
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
bucket, object = trimVolume(bucket), trimVolume(object)
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
@@ -68,6 +69,7 @@ func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string)
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
bucket, object = trimVolume(bucket), trimVolume(object)
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
@@ -114,6 +116,7 @@ func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, va
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
bucket, object = trimVolume(bucket), trimVolume(object)
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
@@ -135,6 +138,7 @@ func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
// ListAttributes lists all attributes for an object or a bucket.
func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
bucket, object = trimVolume(bucket), trimVolume(object)
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
@@ -160,6 +164,7 @@ func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
// When object is empty the entire bucket sidecar directory is removed,
// cleaning up any orphaned object or multipart metadata within it.
func (s SideCar) DeleteAttributes(bucket, object string) error {
bucket, object = trimVolume(bucket), trimVolume(object)
if object == "" {
// Remove the entire bucket sidecar directory so that orphaned
// object/multipart metadata does not accumulate after DeleteBucket.
@@ -184,8 +189,9 @@ func (s SideCar) DeleteAttributes(bucket, object string) error {
// newObject so that path-based lookups continue to work after the data
// directory has been renamed.
func (s SideCar) RenameObject(bucket, oldObject, newObject string) error {
oldPath := filepath.Join(s.dir, bucket, oldObject)
newPath := filepath.Join(s.dir, bucket, newObject)
bucket = trimVolume(bucket)
oldPath := filepath.Join(s.dir, bucket, trimVolume(oldObject))
newPath := filepath.Join(s.dir, bucket, trimVolume(newObject))
if err := os.MkdirAll(filepath.Dir(newPath), 0777); err != nil {
if errors.Is(err, syscall.ENOSPC) {
@@ -14,12 +14,7 @@
//go:build !windows
package posix
package meta
import (
"github.com/versity/versitygw/s3err"
)
func handleParentDirError(_ string) error {
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
// trimVolume is a no-op on non-Windows platforms.
func trimVolume(p string) string { return p }
@@ -14,33 +14,16 @@
//go:build windows
package posix
package meta
import (
"os"
"path/filepath"
"github.com/versity/versitygw/s3err"
"strings"
)
func handleParentDirError(name string) error {
dir := filepath.Dir(name)
// Walk up the directory hierarchy
for dir != "." && dir != "/" {
d, statErr := os.Stat(dir)
if statErr == nil {
// Path component exists
if !d.IsDir() {
// Found a file in the ancestor path
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
// Found a valid directory ancestor, parent truly doesn't exist
break
}
// Continue checking parent directories
dir = filepath.Dir(dir)
}
// Parent doesn't exist or is a directory, treat as ENOENT
return nil
// trimVolume strips the Windows drive letter (e.g. "C:") and any leading
// path separator from p so it can be safely used as a sub-path component
// inside filepath.Join without becoming an absolute root.
func trimVolume(p string) string {
return strings.TrimLeft(p[len(filepath.VolumeName(p)):], `\/`)
}
+54
View File
@@ -0,0 +1,54 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build !windows
package posix
import (
"errors"
"os"
"syscall"
"github.com/versity/versitygw/s3err"
)
func handleParentDirError(_ string) error {
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
// isErrNotDir reports whether err indicates that a path component is a file,
// not a directory (POSIX ENOTDIR).
func isErrNotDir(err error) bool {
return errors.Is(err, syscall.ENOTDIR)
}
// isErrNameTooLong reports whether err indicates that a filename or path
// component is too long (POSIX ENAMETOOLONG).
func isErrNameTooLong(err error) bool {
return errors.Is(err, syscall.ENAMETOOLONG)
}
// isErrDirNotEmpty reports whether err indicates that a directory is not empty
// (POSIX ENOTEMPTY).
func isErrDirNotEmpty(err error) bool {
return errors.Is(err, syscall.ENOTEMPTY)
}
// openForRead opens a file for reading. On non-Windows systems, os.Open is
// sufficient because POSIX allows removing (unlinking) a file that is still
// open by another process.
func openForRead(name string) (*os.File, error) {
return os.Open(name)
}
+131
View File
@@ -0,0 +1,131 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build windows
package posix
import (
"errors"
"os"
"path/filepath"
"syscall"
"github.com/versity/versitygw/s3err"
)
func handleParentDirError(name string) error {
dir := filepath.Dir(name)
// Walk up the directory hierarchy
for dir != "." && dir != "/" {
d, statErr := os.Stat(dir)
if statErr == nil {
// Path component exists
if !d.IsDir() {
// Found a file in the ancestor path
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
// Found a valid directory ancestor, parent truly doesn't exist
break
}
// Continue checking parent directories
dir = filepath.Dir(dir)
}
// Parent doesn't exist or is a directory, treat as ENOENT
return nil
}
// errDirectory is Windows ERROR_DIRECTORY (267): "The directory name is invalid."
// Windows returns this when opening a path like "file/" where "file" is a regular
// file rather than a directory — the POSIX equivalent is ENOTDIR.
const errDirectory = syscall.Errno(267)
// errInvalidName is Windows ERROR_INVALID_NAME (123): "The filename, directory
// name, or volume label syntax is incorrect." Windows returns this when a path
// component exceeds the filesystem name-length limit — the POSIX equivalent is
// ENAMETOOLONG.
const errInvalidName = syscall.Errno(123)
// errDirNotEmpty is Windows ERROR_DIR_NOT_EMPTY (145): "The directory is not
// empty." — the POSIX equivalent is ENOTEMPTY.
const errDirNotEmpty = syscall.Errno(145)
// isErrNameTooLong reports whether err indicates that a filename or path
// component is too long. On Windows this covers both ENAMETOOLONG
// (ERROR_FILENAME_EXCED_RANGE, 206) and ERROR_INVALID_NAME (123), which is
// what the Windows kernel returns for a 300-character filename that exceeds
// MAX_PATH.
func isErrNameTooLong(err error) bool {
if errors.Is(err, syscall.ENAMETOOLONG) {
return true
}
var sysErr syscall.Errno
if errors.As(err, &sysErr) {
return sysErr == errInvalidName
}
return false
}
// isErrDirNotEmpty reports whether err indicates that a directory is not empty.
// On Windows this covers both ENOTEMPTY and ERROR_DIR_NOT_EMPTY (145).
func isErrDirNotEmpty(err error) bool {
if errors.Is(err, syscall.ENOTEMPTY) {
return true
}
var sysErr syscall.Errno
if errors.As(err, &sysErr) {
return sysErr == errDirNotEmpty
}
return false
}
// isErrNotDir reports whether err indicates that a path component is a file,
// not a directory. On Windows this covers both ENOTDIR and ERROR_DIRECTORY
// because os.Open / os.Stat do not map ERROR_DIRECTORY to ENOTDIR.
func isErrNotDir(err error) bool {
if errors.Is(err, syscall.ENOTDIR) {
return true
}
var sysErr syscall.Errno
if errors.As(err, &sysErr) {
return sysErr == errDirectory
}
return false
}
// openForRead opens a file for reading with FILE_SHARE_DELETE so that a
// concurrent DeleteObject (os.Remove) can succeed even while the file handle
// is held open for streaming the GET response body. Without this flag,
// Windows returns "The process cannot access the file because it is being
// used by another process" on the Remove call.
func openForRead(name string) (*os.File, error) {
ptr, err := syscall.UTF16PtrFromString(name)
if err != nil {
return nil, &os.PathError{Op: "open", Path: name, Err: err}
}
h, err := syscall.CreateFile(
ptr,
syscall.GENERIC_READ,
syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE,
nil,
syscall.OPEN_EXISTING,
syscall.FILE_ATTRIBUTE_NORMAL,
0,
)
if err != nil {
return nil, &os.PathError{Op: "open", Path: name, Err: err}
}
return os.NewFile(uintptr(h), name), nil
}
+54 -47
View File
@@ -372,7 +372,7 @@ func (p *Posix) doesBucketAndObjectExist(bucket, object string) error {
}
_, err = os.Stat(filepath.Join(bucket, object))
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -1060,7 +1060,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error {
// data file simply doesn't exist — the two cases are indistinguishable
// from metadata alone. Verify the data file directly so callers
// receive the correct NoSuchVersion / NoSuchKey error.
if _, statErr := os.Stat(filepath.Join(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) {
if _, statErr := os.Stat(filepath.Join(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) {
if versionId != "" {
return s3err.GetAPIError(s3err.ErrNoSuchVersion)
}
@@ -1068,7 +1068,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error {
}
_, err := p.meta.RetrieveAttribute(nil, bucket, object, deleteMarkerKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -1087,7 +1087,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error {
// Check if the given object is a delete marker
func (p *Posix) isObjDeleteMarker(bucket, object string) (bool, error) {
_, err := p.meta.RetrieveAttribute(nil, bucket, object, deleteMarkerKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return false, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, meta.ErrNoSuchKey) {
@@ -3289,7 +3289,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyPartResult{}, s3err.GetNoSuchUploadErr(*upi.UploadId)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(*upi.Key)), 1024)
}
if err != nil {
@@ -3360,7 +3360,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
}
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(srcObject)), 1024)
}
if err != nil {
@@ -3776,10 +3776,10 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
_ = p.meta.DeleteAttribute(*po.Bucket, *po.Key, objectRetentionKey)
}
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return s3response.PutObjectOutput{}, s3err.GetKeyTooLongErr(int64(len(*po.Key)), 1024)
}
if errors.Is(err, syscall.ENOTDIR) {
if isErrNotDir(err) {
parentErr := handleParentDirError(name)
if parentErr != nil {
return s3response.PutObjectOutput{}, parentErr
@@ -4067,11 +4067,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
if getString(input.VersionId) == "" {
// if the versionId is not specified, make the current version a delete marker
fi, err := os.Stat(objpath)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
// AWS returns success if the object does not exist
return &s3.DeleteObjectOutput{}, nil
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
}
if err != nil {
@@ -4134,7 +4134,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
versionPath := p.genObjVersionPath(bucket, object)
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
// AWS returns success if the object does not exist
return &s3.DeleteObjectOutput{
VersionId: input.VersionId,
@@ -4151,7 +4151,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
// but "foo" is a regular file (not a directory), the path cannot
// contain any object.
_, statErr := os.Stat(filepath.Join(bucket, object))
if errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) {
if errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) {
return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil
}
vId = []byte(nullVersionId)
@@ -4204,7 +4204,6 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
if err != nil {
return nil, fmt.Errorf("open obj version: %w", err)
}
defer sf.Close()
acct, ok := ctx.Value("account").(auth.Account)
if !ok {
acct = auth.Account{}
@@ -4220,9 +4219,14 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
_, err = io.Copy(f, sf)
if err != nil {
_ = sf.Close()
return nil, fmt.Errorf("copy object %w", err)
}
if err := sf.Close(); err != nil {
return nil, fmt.Errorf("close obj version: %w", err)
}
if err := f.link(); err != nil {
return nil, fmt.Errorf("link tmp file: %w", err)
}
@@ -4275,10 +4279,10 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId)
err = os.Remove(filepath.Join(versionPath, *input.VersionId))
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
}
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, *input.VersionId)
}
if err != nil {
@@ -4296,10 +4300,10 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
}
fi, err := os.Stat(objpath)
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
}
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
// AWS returns success if the object does not exist
return &s3.DeleteObjectOutput{}, nil
}
@@ -4329,7 +4333,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENOTEMPTY) {
if isErrDirNotEmpty(err) {
// If the directory object has been uploaded explicitly
// remove the directory object (remove the ETag)
_, err = p.meta.RetrieveAttribute(nil, objpath, "", etagkey)
@@ -4489,7 +4493,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
object := *input.Key
if versionId != "" {
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -4508,13 +4512,13 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
objPath := filepath.Join(bucket, object)
fid, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetNoSuchVersionErr(*input.Key, versionId)
}
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
}
if err != nil {
@@ -4648,7 +4652,10 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
versionId = string(vId)
}
f, err := os.Open(objPath)
// openForRead opens with FILE_SHARE_DELETE on Windows so that a concurrent
// DeleteObject can call os.Remove on this file while the GET response body
// is still being streamed. On POSIX, os.Open is sufficient.
f, err := openForRead(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
@@ -4657,13 +4664,13 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
}
fi, err := f.Stat()
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
}
if err != nil {
@@ -4829,7 +4836,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
if versionId != "" {
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -4849,13 +4856,13 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetNoSuchVersionErr(*input.Key, versionId)
}
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
}
if err != nil {
@@ -5174,7 +5181,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
return s3response.CopyObjectOutput{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, srcVersionId)
}
vId, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -5197,13 +5204,13 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
objPath := joinPathWithTrailer(srcBucket, srcObject)
f, err := os.Open(objPath)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if p.versioningEnabled() && vEnabled {
return s3response.CopyObjectOutput{}, s3err.GetNoSuchVersionErr(origSrcObject, srcVersionId)
}
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return s3response.CopyObjectOutput{}, s3err.GetKeyTooLongErr(int64(len(origSrcObject)), 1024)
}
if err != nil {
@@ -5344,7 +5351,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
b, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, etagkey)
etag = string(b)
vId, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
version = backend.GetPtrFromString(string(vId))
@@ -5888,10 +5895,10 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
if versionId == "" {
_, err = os.Stat(filepath.Join(bucket, object))
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
}
if err != nil {
@@ -5905,7 +5912,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -5929,7 +5936,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
func (p *Posix) getAttrTags(bucket, object, versionId string) (map[string]string, error) {
tags := make(map[string]string)
b, err := p.meta.RetrieveAttribute(nil, bucket, object, tagHdr)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -5978,10 +5985,10 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
if versionId == "" {
_, err = os.Stat(filepath.Join(bucket, object))
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return s3err.GetAPIError(s3err.ErrKeyTooLong)
}
if err != nil {
@@ -5995,7 +6002,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -6015,7 +6022,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
if tags == nil {
err = p.meta.DeleteAttribute(bucket, object, tagHdr)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -6036,7 +6043,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
}
err = p.meta.StoreAttribute(nil, bucket, object, tagHdr, b)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -6420,7 +6427,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI
return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -6439,7 +6446,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI
}
err = p.meta.StoreAttribute(nil, bucket, object, objectLegalHoldKey, statusData)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -6481,7 +6488,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -6500,7 +6507,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI
}
data, err := p.meta.RetrieveAttribute(nil, bucket, object, objectLegalHoldKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -6547,7 +6554,7 @@ func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionI
return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -6602,7 +6609,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -6621,7 +6628,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI
}
data, err := p.meta.RetrieveAttribute(nil, bucket, object, objectRetentionKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetNoSuchVersionErr(object, versionId)
}
+6 -5
View File
@@ -109,7 +109,7 @@ func (tmp *tmpfile) link() error {
backoffMs := initialBackoffMs
for {
err = backend.MoveFile(tempname, objPath, defaultFilePerm)
if !errors.Is(err, syscall.ENOENT) {
if !os.IsNotExist(err) {
break
}
// The parent directory may have been concurrently removed; backoff and retry.
@@ -117,11 +117,12 @@ func (tmp *tmpfile) link() error {
sleepWithJitter(backoffMs)
backoffMs = min((backoffMs * 2), maxBackoffMs)
err = backend.MkdirAll(filepath.Dir(objPath), tmp.uid, tmp.gid,
// Best-effort: recreate the parent directory. Ignore errors here;
// if recreation fails transiently (e.g. Windows pending-delete on
// a recently removed directory), the next MoveFile attempt will
// return os.IsNotExist again and we will retry.
_ = backend.MkdirAll(filepath.Dir(objPath), tmp.uid, tmp.gid,
tmp.doChown, tmp.newDirPerm)
if err != nil {
return fmt.Errorf("recreate parent dir: %w", err)
}
}
return err
}
+27
View File
@@ -44,6 +44,7 @@ var (
azureTests bool
tlsStatus bool
parallel bool
windowsTests bool
sidecarTests bool
)
@@ -121,6 +122,12 @@ func initTestCommands() []*cli.Command {
Destination: &azureTests,
Aliases: []string{"azure"},
},
&cli.BoolFlag{
Name: "windows-test-mode",
Usage: "Skips tests that are not supported on Windows",
Destination: &windowsTests,
Aliases: []string{"windows"},
},
&cli.BoolFlag{
Name: "sidecar-test-mode",
Usage: "Skips tests that are not supported by Sidecar",
@@ -176,6 +183,12 @@ func initTestCommands() []*cli.Command {
Destination: &versioningEnabled,
Aliases: []string{"vs"},
},
&cli.BoolFlag{
Name: "windows-test-mode",
Usage: "Skips tests that are not supported on Windows",
Destination: &windowsTests,
Aliases: []string{"windows"},
},
},
},
{
@@ -421,6 +434,10 @@ func getAction(tf testFunc) func(ctx *cli.Context) error {
if azureTests {
opts = append(opts, integration.WithAzureMode())
}
if windowsTests {
opts = append(opts, integration.WithWindowsMode())
opts = append(opts, integration.WithSidecarMode())
}
if sidecarTests {
opts = append(opts, integration.WithSidecarMode())
}
@@ -470,6 +487,10 @@ func extractIntTests() (commands []*cli.Command) {
if azureTests {
opts = append(opts, integration.WithAzureMode())
}
if windowsTests {
opts = append(opts, integration.WithWindowsMode())
opts = append(opts, integration.WithSidecarMode())
}
if sidecarTests {
opts = append(opts, integration.WithSidecarMode())
}
@@ -497,6 +518,12 @@ func extractIntTests() (commands []*cli.Command) {
Destination: &sidecarTests,
Aliases: []string{"sidecar"},
},
&cli.BoolFlag{
Name: "windows-test-mode",
Usage: "Skips tests that are not supported on Windows",
Destination: &windowsTests,
Aliases: []string{"windows"},
},
},
})
}
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env pwsh
# Copyright 2026 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# PowerShell equivalent of runtests.sh for Windows
$ErrorActionPreference = "Stop"
# Temp directories
$tmpGw = Join-Path $env:TEMP "gw"
$tmpCovdata = Join-Path $env:TEMP "covdata"
$tmpHttpsCovdata = Join-Path $env:TEMP "https.covdata"
$tmpVersioningCovdata = Join-Path $env:TEMP "versioning.covdata"
$tmpVersioningHttpsCovdata = Join-Path $env:TEMP "versioning.https.covdata"
$tmpNoaclCovdata = Join-Path $env:TEMP "noacl.covdata"
$tmpVersioningDir = Join-Path $env:TEMP "versioningdir"
$tmpSidecar = Join-Path $env:TEMP "sidecar"
foreach ($dir in @($tmpGw, $tmpCovdata, $tmpHttpsCovdata, $tmpVersioningCovdata,
$tmpVersioningHttpsCovdata, $tmpNoaclCovdata, $tmpVersioningDir,
$tmpSidecar)) {
if (Test-Path $dir) { Remove-Item -Recurse -Force $dir }
New-Item -ItemType Directory -Path $dir | Out-Null
}
# Setup TLS certificate and key
Write-Host "Generating TLS certificate and key in the cert.pem and key.pem files"
openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048
if ($LASTEXITCODE -ne 0) { throw "Failed to generate private key" }
openssl req -new -x509 -key key.pem -out cert.pem -days 365 `
-subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
if ($LASTEXITCODE -ne 0) { throw "Failed to generate certificate" }
function Start-Gateway {
param(
[string] $CoverDir,
[string[]] $GwArgs
)
$env:GOCOVERDIR = $CoverDir
$proc = Start-Process -FilePath ".\versitygw.exe" -ArgumentList $GwArgs -PassThru -NoNewWindow
Remove-Item Env:\GOCOVERDIR -ErrorAction SilentlyContinue
return $proc
}
function Invoke-GwTest {
param(
[string] $Description,
[string[]] $TestArgs,
[System.Diagnostics.Process] $GatewayProc
)
& .\versitygw.exe test @TestArgs
if ($LASTEXITCODE -ne 0) {
Write-Host "$Description failed"
Stop-Process -Id $GatewayProc.Id -Force -ErrorAction SilentlyContinue
exit 1
}
}
# ---------------------------------------------------------------------------
# 1. HTTP (port 7070)
# ---------------------------------------------------------------------------
Write-Host "Running the sdk test over http"
$gwProc = Start-Gateway -CoverDir $tmpCovdata `
-GwArgs @("-a", "user", "-s", "pass", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw)
Start-Sleep -Seconds 1
if ($gwProc.HasExited) {
Write-Host "server no longer running"
exit 1
}
Invoke-GwTest -Description "full flow tests" -GatewayProc $gwProc `
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "full-flow", "--parallel", "--windows-test-mode")
Invoke-GwTest -Description "posix tests" -GatewayProc $gwProc `
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "posix", "--windows-test-mode")
Invoke-GwTest -Description "iam tests" -GatewayProc $gwProc `
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "iam")
Stop-Process -Id $gwProc.Id -Force -ErrorAction SilentlyContinue
# ---------------------------------------------------------------------------
# 2. HTTPS (port 7071)
# ---------------------------------------------------------------------------
Write-Host "Running the sdk test over https"
$gwHttpsProc = Start-Gateway -CoverDir $tmpHttpsCovdata `
-GwArgs @("--cert", "$PWD\cert.pem", "--key", "$PWD\key.pem",
"-p", ":7071", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw)
Start-Sleep -Seconds 1
if ($gwHttpsProc.HasExited) {
Write-Host "https server no longer running"
exit 1
}
Invoke-GwTest -Description "https full flow tests" -GatewayProc $gwHttpsProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "full-flow", "--parallel", "--windows-test-mode")
Invoke-GwTest -Description "https posix tests" -GatewayProc $gwHttpsProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "posix", "--windows-test-mode")
Invoke-GwTest -Description "https iam tests" -GatewayProc $gwHttpsProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "iam")
Stop-Process -Id $gwHttpsProc.Id -Force -ErrorAction SilentlyContinue
# ---------------------------------------------------------------------------
# 3. Versioning HTTP (port 7072)
# ---------------------------------------------------------------------------
Write-Host "Running the sdk test over http against the versioning-enabled gateway"
$gwVsProc = Start-Gateway -CoverDir $tmpVersioningCovdata `
-GwArgs @("-p", ":7072", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw,
"posix", "--sidecar", $tmpSidecar, "--versioning-dir", $tmpVersioningDir, $tmpGw)
Start-Sleep -Seconds 1
if ($gwVsProc.HasExited) {
Write-Host "versioning-enabled server no longer running"
exit 1
}
Invoke-GwTest -Description "versioning-enabled full-flow tests" -GatewayProc $gwVsProc `
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7072", "full-flow", "-vs", "--parallel", "--windows-test-mode")
Invoke-GwTest -Description "versioning-enabled posix tests" -GatewayProc $gwVsProc `
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7072", "posix", "-vs", "--windows-test-mode")
Stop-Process -Id $gwVsProc.Id -Force -ErrorAction SilentlyContinue
# ---------------------------------------------------------------------------
# 4. Versioning HTTPS (port 7073)
# ---------------------------------------------------------------------------
Write-Host "Running the sdk test over https against the versioning-enabled gateway"
$gwVsHttpsProc = Start-Gateway -CoverDir $tmpVersioningHttpsCovdata `
-GwArgs @("--cert", "$PWD\cert.pem", "--key", "$PWD\key.pem",
"-p", ":7073", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw,
"posix", "--sidecar", $tmpSidecar, "--versioning-dir", $tmpVersioningDir, $tmpGw)
Start-Sleep -Seconds 1
if ($gwVsHttpsProc.HasExited) {
Write-Host "versioning-enabled https server no longer running"
exit 1
}
Invoke-GwTest -Description "versioning-enabled https full-flow tests" -GatewayProc $gwVsHttpsProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7073", "full-flow", "-vs", "--parallel", "--windows-test-mode")
Invoke-GwTest -Description "versioning-enabled https posix tests" -GatewayProc $gwVsHttpsProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7073", "posix", "-vs", "--windows-test-mode")
Stop-Process -Id $gwVsHttpsProc.Id -Force -ErrorAction SilentlyContinue
# ---------------------------------------------------------------------------
# 5. No ACL (port 7074)
# ---------------------------------------------------------------------------
Write-Host "Running No ACL integration tests"
$gwNoAclProc = Start-Gateway -CoverDir $tmpNoaclCovdata `
-GwArgs @("-p", ":7074", "-a", "user", "-s", "pass", "-noacl", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw)
Start-Sleep -Seconds 1
if ($gwNoAclProc.HasExited) {
Write-Host "noacl server no longer running"
exit 1
}
Invoke-GwTest -Description "No ACL integration tests" -GatewayProc $gwNoAclProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7074", "noacl")
Stop-Process -Id $gwNoAclProc.Id -Force -ErrorAction SilentlyContinue
+76 -19
View File
@@ -17,6 +17,7 @@ package integration
import (
"context"
"fmt"
"sort"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -90,6 +91,29 @@ func ListBuckets_as_user(s *S3Conf) error {
func ListBuckets_as_admin(s *S3Conf) error {
testName := "ListBuckets_as_admin"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
testuser, adminUser := getUser("user"), getUser("admin")
err := createUsers(s, []user{testuser, adminUser})
if err != nil {
return err
}
adminClient := s.getUserClient(adminUser)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
initOut, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err != nil {
return err
}
ignore := make([]string, 0, len(initOut.Buckets))
for _, b := range initOut.Buckets {
if *b.Name != bucket {
ignore = append(ignore, *b.Name)
}
}
buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
for range 6 {
bckt := getBucketName()
@@ -104,12 +128,6 @@ func ListBuckets_as_admin(s *S3Conf) error {
BucketRegion: &s.awsRegion,
})
}
testuser, adminUser := getUser("user"), getUser("admin")
err := createUsers(s, []user{testuser, adminUser})
if err != nil {
return err
}
bckts := []string{}
for i := range 3 {
@@ -121,9 +139,7 @@ func ListBuckets_as_admin(s *S3Conf) error {
return err
}
adminClient := s.getUserClient(adminUser)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err != nil {
@@ -134,7 +150,7 @@ func ListBuckets_as_admin(s *S3Conf) error {
return fmt.Errorf("expected buckets owner to be %v, instead got %v",
adminUser.access, getString(out.Owner.ID))
}
if !compareBuckets(out.Buckets, buckets) {
if !compareBuckets(out.Buckets, buckets, ignore...) {
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
sprintBuckets(buckets), sprintBuckets(out.Buckets))
}
@@ -154,6 +170,21 @@ func ListBuckets_with_prefix(s *S3Conf) error {
testName := "ListBuckets_with_prefix"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
prefix := "my-prefix-"
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
prefixInitOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
Prefix: &prefix,
})
cancel()
if err != nil {
return err
}
ignore := make([]string, 0, len(prefixInitOut.Buckets))
for _, b := range prefixInitOut.Buckets {
ignore = append(ignore, *b.Name)
}
allBuckets, prefixedBuckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}, []types.Bucket{}
for i := range 5 {
bckt := getBucketName()
@@ -179,7 +210,7 @@ func ListBuckets_with_prefix(s *S3Conf) error {
}
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
Prefix: &prefix,
})
@@ -196,7 +227,7 @@ func ListBuckets_with_prefix(s *S3Conf) error {
return fmt.Errorf("expected prefix to be %v, instead got %v",
prefix, getString(out.Prefix))
}
if !compareBuckets(out.Buckets, prefixedBuckets) {
if !compareBuckets(out.Buckets, prefixedBuckets, ignore...) {
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
prefixedBuckets, out.Buckets)
}
@@ -243,7 +274,14 @@ func ListBuckets_invalid_max_buckets(s *S3Conf) error {
func ListBuckets_truncated(s *S3Conf) error {
testName := "ListBuckets_truncated"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err != nil {
return err
}
newBuckets := []types.Bucket{}
for range 5 {
bckt := getBucketName()
@@ -252,14 +290,19 @@ func ListBuckets_truncated(s *S3Conf) error {
return err
}
buckets = append(buckets, types.Bucket{
newBuckets = append(newBuckets, types.Bucket{
Name: &bckt,
BucketRegion: &s.awsRegion,
})
}
buckets := append(initOut.Buckets, newBuckets...)
sort.Slice(buckets, func(i, j int) bool {
return *buckets[i].Name < *buckets[j].Name
})
maxBuckets := int32(3)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
MaxBuckets: &maxBuckets,
})
@@ -292,7 +335,7 @@ func ListBuckets_truncated(s *S3Conf) error {
if !compareBuckets(out.Buckets, buckets[maxBuckets:]) {
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
sprintBuckets(buckets[:maxBuckets]), sprintBuckets(out.Buckets))
sprintBuckets(buckets[maxBuckets:]), sprintBuckets(out.Buckets))
}
if out.ContinuationToken != nil {
return fmt.Errorf("expected nil continuation token, instead got %v",
@@ -302,7 +345,7 @@ func ListBuckets_truncated(s *S3Conf) error {
return fmt.Errorf("expected nil prefix, instead got %v", *out.Prefix)
}
for _, elem := range buckets[1:] {
for _, elem := range newBuckets {
err = teardown(s, *elem.Name)
if err != nil {
return err
@@ -335,6 +378,20 @@ func ListBuckets_empty_success(s *S3Conf) error {
func ListBuckets_success(s *S3Conf) error {
testName := "ListBuckets_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err != nil {
return err
}
ignore := make([]string, 0, len(initOut.Buckets))
for _, b := range initOut.Buckets {
if *b.Name != bucket {
ignore = append(ignore, *b.Name)
}
}
buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
for range 5 {
bckt := getBucketName()
@@ -350,7 +407,7 @@ func ListBuckets_success(s *S3Conf) error {
})
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err != nil {
@@ -361,7 +418,7 @@ func ListBuckets_success(s *S3Conf) error {
return fmt.Errorf("expected owner to be %v, instead got %v",
s.awsID, getString(out.Owner.ID))
}
if !compareBuckets(out.Buckets, buckets) {
if !compareBuckets(out.Buckets, buckets, ignore...) {
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
sprintBuckets(buckets), sprintBuckets(out.Buckets))
}
+34 -11
View File
@@ -21,6 +21,7 @@ import (
"encoding/base64"
"fmt"
"net/http"
"sort"
"strings"
"time"
@@ -52,10 +53,21 @@ func PutObject_special_chars(s *S3Conf) error {
"my?key", "my^key", "my{}key", "my%key", "my`key",
"my[]key", "my~key", "my<>key", "my|key", "my#key",
}
if !s.azureTests {
// azure currently can't handle backslashes in object names
if !s.azureTests && !s.windowsTests {
// azure and windows cannot handle backslashes in object names:
// on Windows, '\' is a path separator so 'my\key' is stored as 'my/key'
objnames = append(objnames, "my\\key")
}
if s.windowsTests {
// ':', '?', '<', '>', '|', '*' are not valid filename characters on Windows
var filtered []string
for _, name := range objnames {
if !strings.ContainsAny(name, `:?<>|*`) {
filtered = append(filtered, name)
}
}
objnames = filtered
}
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs, err := putObjects(s3client, objnames, bucket)
@@ -1169,19 +1181,30 @@ func PutObject_false_negative_object_names(s *S3Conf) error {
testName := "PutObject_false_negative_object_names"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []string{
"%252e%252e%252fetc/passwd", // double encoding
"%2e%2e/%2e%2e/%2e%2e/.ssh/id_rsa", // double URL-encoded
"%u002e%u002e/%u002e%u002e/etc/passwd", // unicode escape
"..%2f..%2f..%2fsecret/file.txt", // URL-encoded
"..%c0%af..%c0%afetc/passwd", // UTF-8 overlong trick
".../.../.../target.txt",
"..\\u2215..\\u2215etc/passwd", // Unicode division slash
"dir/%20../file.txt", // encoded space
"%252e%252e%252fetc/passwd", // double encoding
"%2e%2e/%2e%2e/%2e%2e/.ssh/id_rsa", // double URL-encoded
"%u002e%u002e/%u002e%u002e/etc/passwd", // unicode escape
"..%2f..%2f..%2fsecret/file.txt", // URL-encoded
"..%c0%af..%c0%afetc/passwd", // UTF-8 overlong trick
"dir/%c0%ae%c0%ae/%c0%ae%c0%ae/etc/passwd", // overlong UTF-8 encoding
"logs/latest -> /etc/passwd", // symlink attacks
//TODO: add this test case in advanced routing
// "/etc/passwd" // absolute path injection
}
if !s.windowsTests {
// Windows strips trailing dots from path components, making '...' an
// invalid directory name (the filesystem rejects MkdirAll with it).
objs = append(objs, ".../.../.../target.txt")
objs = append(objs, "dir/%20../file.txt") // encoded space
// literal backslashes are treated as path separators on Windows
objs = append(objs, "..\\u2215..\\u2215etc/passwd") // Unicode division slash
// On Windows, '>' is an invalid filename character. The key
// 'logs/latest -> /etc/passwd' creates a directory component
// 'latest -> ' (containing '>') which MkdirAll rejects.
objs = append(objs, "logs/latest -> /etc/passwd") // symlink attacks
}
sort.Strings(objs)
_, err := putObjects(s3client, objs, bucket)
if err != nil {
return err
+19 -7
View File
@@ -196,7 +196,7 @@ func TestPutObject(ts *TestState) {
}
ts.Run(PutObject_success)
ts.Run(PutObject_default_content_type)
if !ts.conf.versioningEnabled {
if !ts.conf.versioningEnabled && !ts.conf.windowsTests {
ts.Run(PutObject_racey_success)
}
ts.Run(PutObject_invalid_credentials)
@@ -957,7 +957,9 @@ func TestPosix(ts *TestState) {
ts.Run(CopyObject_overwrite_same_dir_object)
ts.Run(CopyObject_overwrite_same_file_object)
ts.Run(DeleteObject_directory_not_empty)
ts.Run(PutObject_race_with_delete)
if !ts.conf.windowsTests {
ts.Run(PutObject_race_with_delete)
}
// posix specific versioning tests
if !ts.conf.versioningEnabled {
TestVersioningDisabled(ts)
@@ -1138,10 +1140,14 @@ func TestVersioning(ts *TestState) {
ts.Run(Versioning_CopyObject_success)
ts.Run(Versioning_CopyObject_non_existing_version_id)
ts.Run(Versioning_CopyObject_from_an_object_version)
ts.Run(Versioning_CopyObject_special_chars)
if !ts.conf.windowsTests {
ts.Run(Versioning_CopyObject_special_chars)
}
// HeadObject action
ts.Run(Versioning_HeadObject_invalid_versionId)
ts.Run(Versioning_HeadObject_non_existing_object_version)
if !ts.conf.windowsTests {
ts.Run(Versioning_HeadObject_non_existing_object_version)
}
ts.Run(Versioning_HeadObject_invalid_parent)
ts.Run(Versioning_HeadObject_success)
ts.Run(Versioning_HeadObject_without_versionId)
@@ -1170,7 +1176,9 @@ func TestVersioning(ts *TestState) {
ts.Run(Versioning_DeleteObject_invalid_versionId)
ts.Run(Versioning_DeleteObject_delete_object_version)
ts.Run(Versioning_DeleteObject_non_existing_object)
ts.Run(Versioning_DeleteObject_delete_a_delete_marker)
if !ts.conf.windowsTests {
ts.Run(Versioning_DeleteObject_delete_a_delete_marker)
}
ts.Run(Versioning_Delete_null_versionId_object)
ts.Run(Versioning_DeleteObject_nested_dir_object)
ts.Run(Versioning_DeleteObject_non_existing_objects)
@@ -1221,13 +1229,17 @@ func TestVersioning(ts *TestState) {
ts.Run(Versioning_WORM_PutObject_overwrite_locked_object)
ts.Run(Versioning_WORM_CopyObject_overwrite_locked_object)
ts.Run(Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object)
ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention)
if !ts.conf.windowsTests {
ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention)
}
// Concurrent requests
// Versioninig_concurrent_upload_object
ts.Run(Versioning_AccessControl_GetObjectVersion)
ts.Run(Versioning_AccessControl_HeadObjectVersion)
ts.Run(Versioning_AccessControl_object_tagging_policy)
ts.Run(Versioning_AccessControl_DeleteObject_policy)
if !ts.conf.windowsTests {
ts.Run(Versioning_AccessControl_DeleteObject_policy)
}
ts.Run(Versioning_AccessControl_GetObjectAttributes_policy)
}
+4
View File
@@ -852,6 +852,10 @@ func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error {
testName := "PresignedAuth_Put_GetObject_with_UTF8_chars"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error {
obj := "my-$%^&*;"
if s.windowsTests {
// '*' is not a valid filename character on Windows
obj = "my-$%^&;"
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj})
+4
View File
@@ -46,6 +46,7 @@ type S3Conf struct {
debug bool
versioningEnabled bool
azureTests bool
windowsTests bool
sidecarTests bool
tlsStatus bool
httpClient *http.Client
@@ -121,6 +122,9 @@ func WithVersioningEnabled() Option {
func WithAzureMode() Option {
return func(s *S3Conf) { s.azureTests = true }
}
func WithWindowsMode() Option {
return func(s *S3Conf) { s.windowsTests = true }
}
func WithSidecarMode() Option {
return func(s *S3Conf) { s.sidecarTests = true }
}
+25 -2
View File
@@ -1502,7 +1502,21 @@ func areMapsSame(mp1, mp2 map[string]string) bool {
return true
}
func compareBuckets(list1 []types.Bucket, list2 []types.Bucket) bool {
func compareBuckets(list1 []types.Bucket, list2 []types.Bucket, ignore ...string) bool {
if len(ignore) > 0 {
ignoreSet := make(map[string]struct{}, len(ignore))
for _, name := range ignore {
ignoreSet[name] = struct{}{}
}
filtered := make([]types.Bucket, 0, len(list1))
for _, b := range list1 {
if _, skip := ignoreSet[*b.Name]; !skip {
filtered = append(filtered, b)
}
}
list1 = filtered
}
if len(list1) != len(list2) {
return false
}
@@ -1771,8 +1785,17 @@ func listBuckets(s *S3Conf) error {
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
// randCounter is an atomic counter used by genRandString. It is seeded once
// with time.Now().UnixNano() so that values are unique even when many goroutines
// call genRandString concurrently on systems with low-resolution clocks.
var randCounter = atomic.Uint64{}
func init() {
randCounter.Store(uint64(time.Now().UnixNano()))
}
func genRandString(length int) string {
source := rnd.NewSource(time.Now().UnixNano())
source := rnd.NewSource(int64(randCounter.Add(1)))
random := rnd.New(source)
result := make([]byte, length)
for i := range result {
+7 -1
View File
@@ -20,6 +20,7 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"
@@ -437,12 +438,17 @@ func Versioning_CopyObject_special_chars(s *S3Conf) error {
}
srcObjVersionId := *srcObjVersions[0].VersionId
copySource := fmt.Sprintf("%v/%v?versionId=%v",
bucket,
url.PathEscape(srcObj),
url.QueryEscape(srcObjVersionId),
)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: &bucket,
Key: &dstObj,
CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, srcObj, srcObjVersionId)),
CopySource: getPtr(copySource),
})
cancel()
if err != nil {