Merge pull request #2090 from versity/ben/windows-test

test: add windows build/test for functional tests
This commit is contained in:
Ben McClelland
2026-06-18 13:08:04 -07:00
committed by GitHub
18 changed files with 674 additions and 130 deletions
+38
View File
@@ -0,0 +1,38 @@
name: functional tests (windows)
permissions: {}
on: pull_request
jobs:
build:
name: RunTestsWindows
runs-on: windows-latest
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: 'stable'
id: go
- name: Get Dependencies
run: go mod download
- name: Build
run: |
$version = if (Test-Path VERSION) { Get-Content VERSION } else { git describe --abbrev=0 --tags HEAD }
$build = git rev-parse --short HEAD
$time = (Get-Date -Format "yyyy-MM-dd_hh:mm:sstt")
go build -ldflags "-X=main.Build=$build -X=main.BuildTime=$time -X=main.Version=$version" `
-o versitygw.exe -cover -race ./cmd/versitygw
shell: pwsh
- name: Run Tests
run: pwsh -File .\runtests.ps1
shell: pwsh
- name: Coverage Report
run: go tool covdata percent -i="$env:TEMP\covdata"
shell: pwsh
+3 -4
View File
@@ -10,10 +10,9 @@
### Binary release builds
Download [latest release](https://github.com/versity/versitygw/releases)
| Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
|:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
| ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
| Linux amd64/arm64 | MacOS amd64/arm64 | BSD amd64/arm64 | Windows amd64/arm64 |
|:-----------:|:-----------:|:-----------:|:-----------:|
| ✔️ | ✔️ | ✔️ | ✔️ |
### Use Cases
* Turn your local filesystem into an S3 server with a single command!
* Proxy S3 requests to S3 storage
+8 -2
View File
@@ -49,6 +49,7 @@ func NewSideCar(dir string) (SideCar, error) {
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
bucket, object = trimVolume(bucket), trimVolume(object)
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
@@ -68,6 +69,7 @@ func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string)
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
bucket, object = trimVolume(bucket), trimVolume(object)
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
@@ -114,6 +116,7 @@ func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, va
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
bucket, object = trimVolume(bucket), trimVolume(object)
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
@@ -135,6 +138,7 @@ func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
// ListAttributes lists all attributes for an object or a bucket.
func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
bucket, object = trimVolume(bucket), trimVolume(object)
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
if object == "" {
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
@@ -160,6 +164,7 @@ func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
// When object is empty the entire bucket sidecar directory is removed,
// cleaning up any orphaned object or multipart metadata within it.
func (s SideCar) DeleteAttributes(bucket, object string) error {
bucket, object = trimVolume(bucket), trimVolume(object)
if object == "" {
// Remove the entire bucket sidecar directory so that orphaned
// object/multipart metadata does not accumulate after DeleteBucket.
@@ -184,8 +189,9 @@ func (s SideCar) DeleteAttributes(bucket, object string) error {
// newObject so that path-based lookups continue to work after the data
// directory has been renamed.
func (s SideCar) RenameObject(bucket, oldObject, newObject string) error {
oldPath := filepath.Join(s.dir, bucket, oldObject)
newPath := filepath.Join(s.dir, bucket, newObject)
bucket = trimVolume(bucket)
oldPath := filepath.Join(s.dir, bucket, trimVolume(oldObject))
newPath := filepath.Join(s.dir, bucket, trimVolume(newObject))
if err := os.MkdirAll(filepath.Dir(newPath), 0777); err != nil {
if errors.Is(err, syscall.ENOSPC) {
@@ -14,12 +14,7 @@
//go:build !windows
package posix
package meta
import (
"github.com/versity/versitygw/s3err"
)
func handleParentDirError(_ string) error {
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
// trimVolume is a no-op on non-Windows platforms.
func trimVolume(p string) string { return p }
@@ -14,33 +14,16 @@
//go:build windows
package posix
package meta
import (
"os"
"path/filepath"
"github.com/versity/versitygw/s3err"
"strings"
)
func handleParentDirError(name string) error {
dir := filepath.Dir(name)
// Walk up the directory hierarchy
for dir != "." && dir != "/" {
d, statErr := os.Stat(dir)
if statErr == nil {
// Path component exists
if !d.IsDir() {
// Found a file in the ancestor path
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
// Found a valid directory ancestor, parent truly doesn't exist
break
}
// Continue checking parent directories
dir = filepath.Dir(dir)
}
// Parent doesn't exist or is a directory, treat as ENOENT
return nil
// trimVolume strips the Windows drive letter (e.g. "C:") and any leading
// path separator from p so it can be safely used as a sub-path component
// inside filepath.Join without becoming an absolute root.
func trimVolume(p string) string {
return strings.TrimLeft(p[len(filepath.VolumeName(p)):], `\/`)
}
+54
View File
@@ -0,0 +1,54 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build !windows
package posix
import (
"errors"
"os"
"syscall"
"github.com/versity/versitygw/s3err"
)
func handleParentDirError(_ string) error {
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
// isErrNotDir reports whether err indicates that a path component is a file,
// not a directory (POSIX ENOTDIR).
func isErrNotDir(err error) bool {
return errors.Is(err, syscall.ENOTDIR)
}
// isErrNameTooLong reports whether err indicates that a filename or path
// component is too long (POSIX ENAMETOOLONG).
func isErrNameTooLong(err error) bool {
return errors.Is(err, syscall.ENAMETOOLONG)
}
// isErrDirNotEmpty reports whether err indicates that a directory is not empty
// (POSIX ENOTEMPTY).
func isErrDirNotEmpty(err error) bool {
return errors.Is(err, syscall.ENOTEMPTY)
}
// openForRead opens a file for reading. On non-Windows systems, os.Open is
// sufficient because POSIX allows removing (unlinking) a file that is still
// open by another process.
func openForRead(name string) (*os.File, error) {
return os.Open(name)
}
+131
View File
@@ -0,0 +1,131 @@
// Copyright 2026 Versity Software
// This file is licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//go:build windows
package posix
import (
"errors"
"os"
"path/filepath"
"syscall"
"github.com/versity/versitygw/s3err"
)
func handleParentDirError(name string) error {
dir := filepath.Dir(name)
// Walk up the directory hierarchy
for dir != "." && dir != "/" {
d, statErr := os.Stat(dir)
if statErr == nil {
// Path component exists
if !d.IsDir() {
// Found a file in the ancestor path
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
}
// Found a valid directory ancestor, parent truly doesn't exist
break
}
// Continue checking parent directories
dir = filepath.Dir(dir)
}
// Parent doesn't exist or is a directory, treat as ENOENT
return nil
}
// errDirectory is Windows ERROR_DIRECTORY (267): "The directory name is invalid."
// Windows returns this when opening a path like "file/" where "file" is a regular
// file rather than a directory — the POSIX equivalent is ENOTDIR.
const errDirectory = syscall.Errno(267)
// errInvalidName is Windows ERROR_INVALID_NAME (123): "The filename, directory
// name, or volume label syntax is incorrect." Windows returns this when a path
// component exceeds the filesystem name-length limit — the POSIX equivalent is
// ENAMETOOLONG.
const errInvalidName = syscall.Errno(123)
// errDirNotEmpty is Windows ERROR_DIR_NOT_EMPTY (145): "The directory is not
// empty." — the POSIX equivalent is ENOTEMPTY.
const errDirNotEmpty = syscall.Errno(145)
// isErrNameTooLong reports whether err indicates that a filename or path
// component is too long. On Windows this covers both ENAMETOOLONG
// (ERROR_FILENAME_EXCED_RANGE, 206) and ERROR_INVALID_NAME (123), which is
// what the Windows kernel returns for a 300-character filename that exceeds
// MAX_PATH.
func isErrNameTooLong(err error) bool {
if errors.Is(err, syscall.ENAMETOOLONG) {
return true
}
var sysErr syscall.Errno
if errors.As(err, &sysErr) {
return sysErr == errInvalidName
}
return false
}
// isErrDirNotEmpty reports whether err indicates that a directory is not empty.
// On Windows this covers both ENOTEMPTY and ERROR_DIR_NOT_EMPTY (145).
func isErrDirNotEmpty(err error) bool {
if errors.Is(err, syscall.ENOTEMPTY) {
return true
}
var sysErr syscall.Errno
if errors.As(err, &sysErr) {
return sysErr == errDirNotEmpty
}
return false
}
// isErrNotDir reports whether err indicates that a path component is a file,
// not a directory. On Windows this covers both ENOTDIR and ERROR_DIRECTORY
// because os.Open / os.Stat do not map ERROR_DIRECTORY to ENOTDIR.
func isErrNotDir(err error) bool {
if errors.Is(err, syscall.ENOTDIR) {
return true
}
var sysErr syscall.Errno
if errors.As(err, &sysErr) {
return sysErr == errDirectory
}
return false
}
// openForRead opens a file for reading with FILE_SHARE_DELETE so that a
// concurrent DeleteObject (os.Remove) can succeed even while the file handle
// is held open for streaming the GET response body. Without this flag,
// Windows returns "The process cannot access the file because it is being
// used by another process" on the Remove call.
func openForRead(name string) (*os.File, error) {
ptr, err := syscall.UTF16PtrFromString(name)
if err != nil {
return nil, &os.PathError{Op: "open", Path: name, Err: err}
}
h, err := syscall.CreateFile(
ptr,
syscall.GENERIC_READ,
syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE,
nil,
syscall.OPEN_EXISTING,
syscall.FILE_ATTRIBUTE_NORMAL,
0,
)
if err != nil {
return nil, &os.PathError{Op: "open", Path: name, Err: err}
}
return os.NewFile(uintptr(h), name), nil
}
+54 -47
View File
@@ -372,7 +372,7 @@ func (p *Posix) doesBucketAndObjectExist(bucket, object string) error {
}
_, err = os.Stat(filepath.Join(bucket, object))
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil {
@@ -1060,7 +1060,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error {
// data file simply doesn't exist — the two cases are indistinguishable
// from metadata alone. Verify the data file directly so callers
// receive the correct NoSuchVersion / NoSuchKey error.
if _, statErr := os.Stat(filepath.Join(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) {
if _, statErr := os.Stat(filepath.Join(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) {
if versionId != "" {
return s3err.GetAPIError(s3err.ErrNoSuchVersion)
}
@@ -1068,7 +1068,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error {
}
_, err := p.meta.RetrieveAttribute(nil, bucket, object, deleteMarkerKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -1087,7 +1087,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error {
// Check if the given object is a delete marker
func (p *Posix) isObjDeleteMarker(bucket, object string) (bool, error) {
_, err := p.meta.RetrieveAttribute(nil, bucket, object, deleteMarkerKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return false, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, meta.ErrNoSuchKey) {
@@ -3289,7 +3289,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
if errors.Is(err, fs.ErrNotExist) {
return s3response.CopyPartResult{}, s3err.GetNoSuchUploadErr(*upi.UploadId)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(*upi.Key)), 1024)
}
if err != nil {
@@ -3360,7 +3360,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
}
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(srcObject)), 1024)
}
if err != nil {
@@ -3776,10 +3776,10 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
_ = p.meta.DeleteAttribute(*po.Bucket, *po.Key, objectRetentionKey)
}
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return s3response.PutObjectOutput{}, s3err.GetKeyTooLongErr(int64(len(*po.Key)), 1024)
}
if errors.Is(err, syscall.ENOTDIR) {
if isErrNotDir(err) {
parentErr := handleParentDirError(name)
if parentErr != nil {
return s3response.PutObjectOutput{}, parentErr
@@ -4067,11 +4067,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
if getString(input.VersionId) == "" {
// if the versionId is not specified, make the current version a delete marker
fi, err := os.Stat(objpath)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
// AWS returns success if the object does not exist
return &s3.DeleteObjectOutput{}, nil
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
}
if err != nil {
@@ -4134,7 +4134,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
versionPath := p.genObjVersionPath(bucket, object)
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
// AWS returns success if the object does not exist
return &s3.DeleteObjectOutput{
VersionId: input.VersionId,
@@ -4151,7 +4151,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
// but "foo" is a regular file (not a directory), the path cannot
// contain any object.
_, statErr := os.Stat(filepath.Join(bucket, object))
if errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) {
if errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) {
return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil
}
vId = []byte(nullVersionId)
@@ -4204,7 +4204,6 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
if err != nil {
return nil, fmt.Errorf("open obj version: %w", err)
}
defer sf.Close()
acct, ok := ctx.Value("account").(auth.Account)
if !ok {
acct = auth.Account{}
@@ -4220,9 +4219,14 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
_, err = io.Copy(f, sf)
if err != nil {
_ = sf.Close()
return nil, fmt.Errorf("copy object %w", err)
}
if err := sf.Close(); err != nil {
return nil, fmt.Errorf("close obj version: %w", err)
}
if err := f.link(); err != nil {
return nil, fmt.Errorf("link tmp file: %w", err)
}
@@ -4275,10 +4279,10 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId)
err = os.Remove(filepath.Join(versionPath, *input.VersionId))
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
}
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, *input.VersionId)
}
if err != nil {
@@ -4296,10 +4300,10 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
}
fi, err := os.Stat(objpath)
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
}
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
// AWS returns success if the object does not exist
return &s3.DeleteObjectOutput{}, nil
}
@@ -4329,7 +4333,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENOTEMPTY) {
if isErrDirNotEmpty(err) {
// If the directory object has been uploaded explicitly
// remove the directory object (remove the ETag)
_, err = p.meta.RetrieveAttribute(nil, objpath, "", etagkey)
@@ -4489,7 +4493,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
object := *input.Key
if versionId != "" {
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -4508,13 +4512,13 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
objPath := filepath.Join(bucket, object)
fid, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetNoSuchVersionErr(*input.Key, versionId)
}
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
}
if err != nil {
@@ -4648,7 +4652,10 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
versionId = string(vId)
}
f, err := os.Open(objPath)
// openForRead opens with FILE_SHARE_DELETE on Windows so that a concurrent
// DeleteObject can call os.Remove on this file while the GET response body
// is still being streamed. On POSIX, os.Open is sufficient.
f, err := openForRead(objPath)
if errors.Is(err, fs.ErrNotExist) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
@@ -4657,13 +4664,13 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
}
fi, err := f.Stat()
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
}
if err != nil {
@@ -4829,7 +4836,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
if versionId != "" {
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -4849,13 +4856,13 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
objPath := filepath.Join(bucket, object)
fi, err := os.Stat(objPath)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetNoSuchVersionErr(*input.Key, versionId)
}
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
}
if err != nil {
@@ -5174,7 +5181,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
return s3response.CopyObjectOutput{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, srcVersionId)
}
vId, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -5197,13 +5204,13 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
objPath := joinPathWithTrailer(srcBucket, srcObject)
f, err := os.Open(objPath)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if p.versioningEnabled() && vEnabled {
return s3response.CopyObjectOutput{}, s3err.GetNoSuchVersionErr(origSrcObject, srcVersionId)
}
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return s3response.CopyObjectOutput{}, s3err.GetKeyTooLongErr(int64(len(origSrcObject)), 1024)
}
if err != nil {
@@ -5344,7 +5351,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
b, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, etagkey)
etag = string(b)
vId, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
version = backend.GetPtrFromString(string(vId))
@@ -5888,10 +5895,10 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
if versionId == "" {
_, err = os.Stat(filepath.Join(bucket, object))
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
}
if err != nil {
@@ -5905,7 +5912,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -5929,7 +5936,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
func (p *Posix) getAttrTags(bucket, object, versionId string) (map[string]string, error) {
tags := make(map[string]string)
b, err := p.meta.RetrieveAttribute(nil, bucket, object, tagHdr)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -5978,10 +5985,10 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
if versionId == "" {
_, err = os.Stat(filepath.Join(bucket, object))
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if errors.Is(err, syscall.ENAMETOOLONG) {
if isErrNameTooLong(err) {
return s3err.GetAPIError(s3err.ErrKeyTooLong)
}
if err != nil {
@@ -5995,7 +6002,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -6015,7 +6022,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
if tags == nil {
err = p.meta.DeleteAttribute(bucket, object, tagHdr)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -6036,7 +6043,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
}
err = p.meta.StoreAttribute(nil, bucket, object, tagHdr, b)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -6420,7 +6427,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI
return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -6439,7 +6446,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI
}
err = p.meta.StoreAttribute(nil, bucket, object, objectLegalHoldKey, statusData)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -6481,7 +6488,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -6500,7 +6507,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI
}
data, err := p.meta.RetrieveAttribute(nil, bucket, object, objectLegalHoldKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetNoSuchVersionErr(object, versionId)
}
@@ -6547,7 +6554,7 @@ func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionI
return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -6602,7 +6609,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
}
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
}
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
@@ -6621,7 +6628,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI
}
data, err := p.meta.RetrieveAttribute(nil, bucket, object, objectRetentionKey)
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
if versionId != "" {
return nil, s3err.GetNoSuchVersionErr(object, versionId)
}
+6 -5
View File
@@ -109,7 +109,7 @@ func (tmp *tmpfile) link() error {
backoffMs := initialBackoffMs
for {
err = backend.MoveFile(tempname, objPath, defaultFilePerm)
if !errors.Is(err, syscall.ENOENT) {
if !os.IsNotExist(err) {
break
}
// The parent directory may have been concurrently removed; backoff and retry.
@@ -117,11 +117,12 @@ func (tmp *tmpfile) link() error {
sleepWithJitter(backoffMs)
backoffMs = min((backoffMs * 2), maxBackoffMs)
err = backend.MkdirAll(filepath.Dir(objPath), tmp.uid, tmp.gid,
// Best-effort: recreate the parent directory. Ignore errors here;
// if recreation fails transiently (e.g. Windows pending-delete on
// a recently removed directory), the next MoveFile attempt will
// return os.IsNotExist again and we will retry.
_ = backend.MkdirAll(filepath.Dir(objPath), tmp.uid, tmp.gid,
tmp.doChown, tmp.newDirPerm)
if err != nil {
return fmt.Errorf("recreate parent dir: %w", err)
}
}
return err
}
+27
View File
@@ -44,6 +44,7 @@ var (
azureTests bool
tlsStatus bool
parallel bool
windowsTests bool
sidecarTests bool
)
@@ -121,6 +122,12 @@ func initTestCommands() []*cli.Command {
Destination: &azureTests,
Aliases: []string{"azure"},
},
&cli.BoolFlag{
Name: "windows-test-mode",
Usage: "Skips tests that are not supported on Windows",
Destination: &windowsTests,
Aliases: []string{"windows"},
},
&cli.BoolFlag{
Name: "sidecar-test-mode",
Usage: "Skips tests that are not supported by Sidecar",
@@ -176,6 +183,12 @@ func initTestCommands() []*cli.Command {
Destination: &versioningEnabled,
Aliases: []string{"vs"},
},
&cli.BoolFlag{
Name: "windows-test-mode",
Usage: "Skips tests that are not supported on Windows",
Destination: &windowsTests,
Aliases: []string{"windows"},
},
},
},
{
@@ -421,6 +434,10 @@ func getAction(tf testFunc) func(ctx *cli.Context) error {
if azureTests {
opts = append(opts, integration.WithAzureMode())
}
if windowsTests {
opts = append(opts, integration.WithWindowsMode())
opts = append(opts, integration.WithSidecarMode())
}
if sidecarTests {
opts = append(opts, integration.WithSidecarMode())
}
@@ -470,6 +487,10 @@ func extractIntTests() (commands []*cli.Command) {
if azureTests {
opts = append(opts, integration.WithAzureMode())
}
if windowsTests {
opts = append(opts, integration.WithWindowsMode())
opts = append(opts, integration.WithSidecarMode())
}
if sidecarTests {
opts = append(opts, integration.WithSidecarMode())
}
@@ -497,6 +518,12 @@ func extractIntTests() (commands []*cli.Command) {
Destination: &sidecarTests,
Aliases: []string{"sidecar"},
},
&cli.BoolFlag{
Name: "windows-test-mode",
Usage: "Skips tests that are not supported on Windows",
Destination: &windowsTests,
Aliases: []string{"windows"},
},
},
})
}
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env pwsh
# Copyright 2026 Versity Software
# This file is licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# PowerShell equivalent of runtests.sh for Windows
$ErrorActionPreference = "Stop"
# Temp directories
$tmpGw = Join-Path $env:TEMP "gw"
$tmpCovdata = Join-Path $env:TEMP "covdata"
$tmpHttpsCovdata = Join-Path $env:TEMP "https.covdata"
$tmpVersioningCovdata = Join-Path $env:TEMP "versioning.covdata"
$tmpVersioningHttpsCovdata = Join-Path $env:TEMP "versioning.https.covdata"
$tmpNoaclCovdata = Join-Path $env:TEMP "noacl.covdata"
$tmpVersioningDir = Join-Path $env:TEMP "versioningdir"
$tmpSidecar = Join-Path $env:TEMP "sidecar"
foreach ($dir in @($tmpGw, $tmpCovdata, $tmpHttpsCovdata, $tmpVersioningCovdata,
$tmpVersioningHttpsCovdata, $tmpNoaclCovdata, $tmpVersioningDir,
$tmpSidecar)) {
if (Test-Path $dir) { Remove-Item -Recurse -Force $dir }
New-Item -ItemType Directory -Path $dir | Out-Null
}
# Setup TLS certificate and key
Write-Host "Generating TLS certificate and key in the cert.pem and key.pem files"
openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048
if ($LASTEXITCODE -ne 0) { throw "Failed to generate private key" }
openssl req -new -x509 -key key.pem -out cert.pem -days 365 `
-subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
if ($LASTEXITCODE -ne 0) { throw "Failed to generate certificate" }
function Start-Gateway {
param(
[string] $CoverDir,
[string[]] $GwArgs
)
$env:GOCOVERDIR = $CoverDir
$proc = Start-Process -FilePath ".\versitygw.exe" -ArgumentList $GwArgs -PassThru -NoNewWindow
Remove-Item Env:\GOCOVERDIR -ErrorAction SilentlyContinue
return $proc
}
function Invoke-GwTest {
param(
[string] $Description,
[string[]] $TestArgs,
[System.Diagnostics.Process] $GatewayProc
)
& .\versitygw.exe test @TestArgs
if ($LASTEXITCODE -ne 0) {
Write-Host "$Description failed"
Stop-Process -Id $GatewayProc.Id -Force -ErrorAction SilentlyContinue
exit 1
}
}
# ---------------------------------------------------------------------------
# 1. HTTP (port 7070)
# ---------------------------------------------------------------------------
Write-Host "Running the sdk test over http"
$gwProc = Start-Gateway -CoverDir $tmpCovdata `
-GwArgs @("-a", "user", "-s", "pass", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw)
Start-Sleep -Seconds 1
if ($gwProc.HasExited) {
Write-Host "server no longer running"
exit 1
}
Invoke-GwTest -Description "full flow tests" -GatewayProc $gwProc `
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "full-flow", "--parallel", "--windows-test-mode")
Invoke-GwTest -Description "posix tests" -GatewayProc $gwProc `
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "posix", "--windows-test-mode")
Invoke-GwTest -Description "iam tests" -GatewayProc $gwProc `
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "iam")
Stop-Process -Id $gwProc.Id -Force -ErrorAction SilentlyContinue
# ---------------------------------------------------------------------------
# 2. HTTPS (port 7071)
# ---------------------------------------------------------------------------
Write-Host "Running the sdk test over https"
$gwHttpsProc = Start-Gateway -CoverDir $tmpHttpsCovdata `
-GwArgs @("--cert", "$PWD\cert.pem", "--key", "$PWD\key.pem",
"-p", ":7071", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw)
Start-Sleep -Seconds 1
if ($gwHttpsProc.HasExited) {
Write-Host "https server no longer running"
exit 1
}
Invoke-GwTest -Description "https full flow tests" -GatewayProc $gwHttpsProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "full-flow", "--parallel", "--windows-test-mode")
Invoke-GwTest -Description "https posix tests" -GatewayProc $gwHttpsProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "posix", "--windows-test-mode")
Invoke-GwTest -Description "https iam tests" -GatewayProc $gwHttpsProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "iam")
Stop-Process -Id $gwHttpsProc.Id -Force -ErrorAction SilentlyContinue
# ---------------------------------------------------------------------------
# 3. Versioning HTTP (port 7072)
# ---------------------------------------------------------------------------
Write-Host "Running the sdk test over http against the versioning-enabled gateway"
$gwVsProc = Start-Gateway -CoverDir $tmpVersioningCovdata `
-GwArgs @("-p", ":7072", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw,
"posix", "--sidecar", $tmpSidecar, "--versioning-dir", $tmpVersioningDir, $tmpGw)
Start-Sleep -Seconds 1
if ($gwVsProc.HasExited) {
Write-Host "versioning-enabled server no longer running"
exit 1
}
Invoke-GwTest -Description "versioning-enabled full-flow tests" -GatewayProc $gwVsProc `
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7072", "full-flow", "-vs", "--parallel", "--windows-test-mode")
Invoke-GwTest -Description "versioning-enabled posix tests" -GatewayProc $gwVsProc `
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7072", "posix", "-vs", "--windows-test-mode")
Stop-Process -Id $gwVsProc.Id -Force -ErrorAction SilentlyContinue
# ---------------------------------------------------------------------------
# 4. Versioning HTTPS (port 7073)
# ---------------------------------------------------------------------------
Write-Host "Running the sdk test over https against the versioning-enabled gateway"
$gwVsHttpsProc = Start-Gateway -CoverDir $tmpVersioningHttpsCovdata `
-GwArgs @("--cert", "$PWD\cert.pem", "--key", "$PWD\key.pem",
"-p", ":7073", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw,
"posix", "--sidecar", $tmpSidecar, "--versioning-dir", $tmpVersioningDir, $tmpGw)
Start-Sleep -Seconds 1
if ($gwVsHttpsProc.HasExited) {
Write-Host "versioning-enabled https server no longer running"
exit 1
}
Invoke-GwTest -Description "versioning-enabled https full-flow tests" -GatewayProc $gwVsHttpsProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7073", "full-flow", "-vs", "--parallel", "--windows-test-mode")
Invoke-GwTest -Description "versioning-enabled https posix tests" -GatewayProc $gwVsHttpsProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7073", "posix", "-vs", "--windows-test-mode")
Stop-Process -Id $gwVsHttpsProc.Id -Force -ErrorAction SilentlyContinue
# ---------------------------------------------------------------------------
# 5. No ACL (port 7074)
# ---------------------------------------------------------------------------
Write-Host "Running No ACL integration tests"
$gwNoAclProc = Start-Gateway -CoverDir $tmpNoaclCovdata `
-GwArgs @("-p", ":7074", "-a", "user", "-s", "pass", "-noacl", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw)
Start-Sleep -Seconds 1
if ($gwNoAclProc.HasExited) {
Write-Host "noacl server no longer running"
exit 1
}
Invoke-GwTest -Description "No ACL integration tests" -GatewayProc $gwNoAclProc `
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7074", "noacl")
Stop-Process -Id $gwNoAclProc.Id -Force -ErrorAction SilentlyContinue
+76 -19
View File
@@ -17,6 +17,7 @@ package integration
import (
"context"
"fmt"
"sort"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -90,6 +91,29 @@ func ListBuckets_as_user(s *S3Conf) error {
func ListBuckets_as_admin(s *S3Conf) error {
testName := "ListBuckets_as_admin"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
testuser, adminUser := getUser("user"), getUser("admin")
err := createUsers(s, []user{testuser, adminUser})
if err != nil {
return err
}
adminClient := s.getUserClient(adminUser)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
initOut, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err != nil {
return err
}
ignore := make([]string, 0, len(initOut.Buckets))
for _, b := range initOut.Buckets {
if *b.Name != bucket {
ignore = append(ignore, *b.Name)
}
}
buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
for range 6 {
bckt := getBucketName()
@@ -104,12 +128,6 @@ func ListBuckets_as_admin(s *S3Conf) error {
BucketRegion: &s.awsRegion,
})
}
testuser, adminUser := getUser("user"), getUser("admin")
err := createUsers(s, []user{testuser, adminUser})
if err != nil {
return err
}
bckts := []string{}
for i := range 3 {
@@ -121,9 +139,7 @@ func ListBuckets_as_admin(s *S3Conf) error {
return err
}
adminClient := s.getUserClient(adminUser)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err != nil {
@@ -134,7 +150,7 @@ func ListBuckets_as_admin(s *S3Conf) error {
return fmt.Errorf("expected buckets owner to be %v, instead got %v",
adminUser.access, getString(out.Owner.ID))
}
if !compareBuckets(out.Buckets, buckets) {
if !compareBuckets(out.Buckets, buckets, ignore...) {
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
sprintBuckets(buckets), sprintBuckets(out.Buckets))
}
@@ -154,6 +170,21 @@ func ListBuckets_with_prefix(s *S3Conf) error {
testName := "ListBuckets_with_prefix"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
prefix := "my-prefix-"
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
prefixInitOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
Prefix: &prefix,
})
cancel()
if err != nil {
return err
}
ignore := make([]string, 0, len(prefixInitOut.Buckets))
for _, b := range prefixInitOut.Buckets {
ignore = append(ignore, *b.Name)
}
allBuckets, prefixedBuckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}, []types.Bucket{}
for i := range 5 {
bckt := getBucketName()
@@ -179,7 +210,7 @@ func ListBuckets_with_prefix(s *S3Conf) error {
}
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
Prefix: &prefix,
})
@@ -196,7 +227,7 @@ func ListBuckets_with_prefix(s *S3Conf) error {
return fmt.Errorf("expected prefix to be %v, instead got %v",
prefix, getString(out.Prefix))
}
if !compareBuckets(out.Buckets, prefixedBuckets) {
if !compareBuckets(out.Buckets, prefixedBuckets, ignore...) {
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
prefixedBuckets, out.Buckets)
}
@@ -243,7 +274,14 @@ func ListBuckets_invalid_max_buckets(s *S3Conf) error {
func ListBuckets_truncated(s *S3Conf) error {
testName := "ListBuckets_truncated"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err != nil {
return err
}
newBuckets := []types.Bucket{}
for range 5 {
bckt := getBucketName()
@@ -252,14 +290,19 @@ func ListBuckets_truncated(s *S3Conf) error {
return err
}
buckets = append(buckets, types.Bucket{
newBuckets = append(newBuckets, types.Bucket{
Name: &bckt,
BucketRegion: &s.awsRegion,
})
}
buckets := append(initOut.Buckets, newBuckets...)
sort.Slice(buckets, func(i, j int) bool {
return *buckets[i].Name < *buckets[j].Name
})
maxBuckets := int32(3)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
MaxBuckets: &maxBuckets,
})
@@ -292,7 +335,7 @@ func ListBuckets_truncated(s *S3Conf) error {
if !compareBuckets(out.Buckets, buckets[maxBuckets:]) {
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
sprintBuckets(buckets[:maxBuckets]), sprintBuckets(out.Buckets))
sprintBuckets(buckets[maxBuckets:]), sprintBuckets(out.Buckets))
}
if out.ContinuationToken != nil {
return fmt.Errorf("expected nil continuation token, instead got %v",
@@ -302,7 +345,7 @@ func ListBuckets_truncated(s *S3Conf) error {
return fmt.Errorf("expected nil prefix, instead got %v", *out.Prefix)
}
for _, elem := range buckets[1:] {
for _, elem := range newBuckets {
err = teardown(s, *elem.Name)
if err != nil {
return err
@@ -335,6 +378,20 @@ func ListBuckets_empty_success(s *S3Conf) error {
func ListBuckets_success(s *S3Conf) error {
testName := "ListBuckets_success"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err != nil {
return err
}
ignore := make([]string, 0, len(initOut.Buckets))
for _, b := range initOut.Buckets {
if *b.Name != bucket {
ignore = append(ignore, *b.Name)
}
}
buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
for range 5 {
bckt := getBucketName()
@@ -350,7 +407,7 @@ func ListBuckets_success(s *S3Conf) error {
})
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err != nil {
@@ -361,7 +418,7 @@ func ListBuckets_success(s *S3Conf) error {
return fmt.Errorf("expected owner to be %v, instead got %v",
s.awsID, getString(out.Owner.ID))
}
if !compareBuckets(out.Buckets, buckets) {
if !compareBuckets(out.Buckets, buckets, ignore...) {
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
sprintBuckets(buckets), sprintBuckets(out.Buckets))
}
+34 -11
View File
@@ -21,6 +21,7 @@ import (
"encoding/base64"
"fmt"
"net/http"
"sort"
"strings"
"time"
@@ -52,10 +53,21 @@ func PutObject_special_chars(s *S3Conf) error {
"my?key", "my^key", "my{}key", "my%key", "my`key",
"my[]key", "my~key", "my<>key", "my|key", "my#key",
}
if !s.azureTests {
// azure currently can't handle backslashes in object names
if !s.azureTests && !s.windowsTests {
// azure and windows cannot handle backslashes in object names:
// on Windows, '\' is a path separator so 'my\key' is stored as 'my/key'
objnames = append(objnames, "my\\key")
}
if s.windowsTests {
// ':', '?', '<', '>', '|', '*' are not valid filename characters on Windows
var filtered []string
for _, name := range objnames {
if !strings.ContainsAny(name, `:?<>|*`) {
filtered = append(filtered, name)
}
}
objnames = filtered
}
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs, err := putObjects(s3client, objnames, bucket)
@@ -1169,19 +1181,30 @@ func PutObject_false_negative_object_names(s *S3Conf) error {
testName := "PutObject_false_negative_object_names"
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
objs := []string{
"%252e%252e%252fetc/passwd", // double encoding
"%2e%2e/%2e%2e/%2e%2e/.ssh/id_rsa", // double URL-encoded
"%u002e%u002e/%u002e%u002e/etc/passwd", // unicode escape
"..%2f..%2f..%2fsecret/file.txt", // URL-encoded
"..%c0%af..%c0%afetc/passwd", // UTF-8 overlong trick
".../.../.../target.txt",
"..\\u2215..\\u2215etc/passwd", // Unicode division slash
"dir/%20../file.txt", // encoded space
"%252e%252e%252fetc/passwd", // double encoding
"%2e%2e/%2e%2e/%2e%2e/.ssh/id_rsa", // double URL-encoded
"%u002e%u002e/%u002e%u002e/etc/passwd", // unicode escape
"..%2f..%2f..%2fsecret/file.txt", // URL-encoded
"..%c0%af..%c0%afetc/passwd", // UTF-8 overlong trick
"dir/%c0%ae%c0%ae/%c0%ae%c0%ae/etc/passwd", // overlong UTF-8 encoding
"logs/latest -> /etc/passwd", // symlink attacks
//TODO: add this test case in advanced routing
// "/etc/passwd" // absolute path injection
}
if !s.windowsTests {
// Windows strips trailing dots from path components, making '...' an
// invalid directory name (the filesystem rejects MkdirAll with it).
objs = append(objs, ".../.../.../target.txt")
objs = append(objs, "dir/%20../file.txt") // encoded space
// literal backslashes are treated as path separators on Windows
objs = append(objs, "..\\u2215..\\u2215etc/passwd") // Unicode division slash
// On Windows, '>' is an invalid filename character. The key
// 'logs/latest -> /etc/passwd' creates a directory component
// 'latest -> ' (containing '>') which MkdirAll rejects.
objs = append(objs, "logs/latest -> /etc/passwd") // symlink attacks
}
sort.Strings(objs)
_, err := putObjects(s3client, objs, bucket)
if err != nil {
return err
+19 -7
View File
@@ -196,7 +196,7 @@ func TestPutObject(ts *TestState) {
}
ts.Run(PutObject_success)
ts.Run(PutObject_default_content_type)
if !ts.conf.versioningEnabled {
if !ts.conf.versioningEnabled && !ts.conf.windowsTests {
ts.Run(PutObject_racey_success)
}
ts.Run(PutObject_invalid_credentials)
@@ -957,7 +957,9 @@ func TestPosix(ts *TestState) {
ts.Run(CopyObject_overwrite_same_dir_object)
ts.Run(CopyObject_overwrite_same_file_object)
ts.Run(DeleteObject_directory_not_empty)
ts.Run(PutObject_race_with_delete)
if !ts.conf.windowsTests {
ts.Run(PutObject_race_with_delete)
}
// posix specific versioning tests
if !ts.conf.versioningEnabled {
TestVersioningDisabled(ts)
@@ -1138,10 +1140,14 @@ func TestVersioning(ts *TestState) {
ts.Run(Versioning_CopyObject_success)
ts.Run(Versioning_CopyObject_non_existing_version_id)
ts.Run(Versioning_CopyObject_from_an_object_version)
ts.Run(Versioning_CopyObject_special_chars)
if !ts.conf.windowsTests {
ts.Run(Versioning_CopyObject_special_chars)
}
// HeadObject action
ts.Run(Versioning_HeadObject_invalid_versionId)
ts.Run(Versioning_HeadObject_non_existing_object_version)
if !ts.conf.windowsTests {
ts.Run(Versioning_HeadObject_non_existing_object_version)
}
ts.Run(Versioning_HeadObject_invalid_parent)
ts.Run(Versioning_HeadObject_success)
ts.Run(Versioning_HeadObject_without_versionId)
@@ -1170,7 +1176,9 @@ func TestVersioning(ts *TestState) {
ts.Run(Versioning_DeleteObject_invalid_versionId)
ts.Run(Versioning_DeleteObject_delete_object_version)
ts.Run(Versioning_DeleteObject_non_existing_object)
ts.Run(Versioning_DeleteObject_delete_a_delete_marker)
if !ts.conf.windowsTests {
ts.Run(Versioning_DeleteObject_delete_a_delete_marker)
}
ts.Run(Versioning_Delete_null_versionId_object)
ts.Run(Versioning_DeleteObject_nested_dir_object)
ts.Run(Versioning_DeleteObject_non_existing_objects)
@@ -1221,13 +1229,17 @@ func TestVersioning(ts *TestState) {
ts.Run(Versioning_WORM_PutObject_overwrite_locked_object)
ts.Run(Versioning_WORM_CopyObject_overwrite_locked_object)
ts.Run(Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object)
ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention)
if !ts.conf.windowsTests {
ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention)
}
// Concurrent requests
// Versioninig_concurrent_upload_object
ts.Run(Versioning_AccessControl_GetObjectVersion)
ts.Run(Versioning_AccessControl_HeadObjectVersion)
ts.Run(Versioning_AccessControl_object_tagging_policy)
ts.Run(Versioning_AccessControl_DeleteObject_policy)
if !ts.conf.windowsTests {
ts.Run(Versioning_AccessControl_DeleteObject_policy)
}
ts.Run(Versioning_AccessControl_GetObjectAttributes_policy)
}
+4
View File
@@ -852,6 +852,10 @@ func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error {
testName := "PresignedAuth_Put_GetObject_with_UTF8_chars"
return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error {
obj := "my-$%^&*;"
if s.windowsTests {
// '*' is not a valid filename character on Windows
obj = "my-$%^&;"
}
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj})
+4
View File
@@ -46,6 +46,7 @@ type S3Conf struct {
debug bool
versioningEnabled bool
azureTests bool
windowsTests bool
sidecarTests bool
tlsStatus bool
httpClient *http.Client
@@ -121,6 +122,9 @@ func WithVersioningEnabled() Option {
func WithAzureMode() Option {
return func(s *S3Conf) { s.azureTests = true }
}
func WithWindowsMode() Option {
return func(s *S3Conf) { s.windowsTests = true }
}
func WithSidecarMode() Option {
return func(s *S3Conf) { s.sidecarTests = true }
}
+25 -2
View File
@@ -1502,7 +1502,21 @@ func areMapsSame(mp1, mp2 map[string]string) bool {
return true
}
func compareBuckets(list1 []types.Bucket, list2 []types.Bucket) bool {
func compareBuckets(list1 []types.Bucket, list2 []types.Bucket, ignore ...string) bool {
if len(ignore) > 0 {
ignoreSet := make(map[string]struct{}, len(ignore))
for _, name := range ignore {
ignoreSet[name] = struct{}{}
}
filtered := make([]types.Bucket, 0, len(list1))
for _, b := range list1 {
if _, skip := ignoreSet[*b.Name]; !skip {
filtered = append(filtered, b)
}
}
list1 = filtered
}
if len(list1) != len(list2) {
return false
}
@@ -1771,8 +1785,17 @@ func listBuckets(s *S3Conf) error {
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
// randCounter is an atomic counter used by genRandString. It is seeded once
// with time.Now().UnixNano() so that values are unique even when many goroutines
// call genRandString concurrently on systems with low-resolution clocks.
var randCounter = atomic.Uint64{}
func init() {
randCounter.Store(uint64(time.Now().UnixNano()))
}
func genRandString(length int) string {
source := rnd.NewSource(time.Now().UnixNano())
source := rnd.NewSource(int64(randCounter.Add(1)))
random := rnd.New(source)
result := make([]byte, length)
for i := range result {
+7 -1
View File
@@ -20,6 +20,7 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strings"
"sync"
"time"
@@ -437,12 +438,17 @@ func Versioning_CopyObject_special_chars(s *S3Conf) error {
}
srcObjVersionId := *srcObjVersions[0].VersionId
copySource := fmt.Sprintf("%v/%v?versionId=%v",
bucket,
url.PathEscape(srcObj),
url.QueryEscape(srcObjVersionId),
)
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: &bucket,
Key: &dstObj,
CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, srcObj, srcObjVersionId)),
CopySource: getPtr(copySource),
})
cancel()
if err != nil {