mirror of
https://github.com/versity/versitygw.git
synced 2026-07-02 16:54:25 +00:00
Merge pull request #2090 from versity/ben/windows-test
test: add windows build/test for functional tests
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
name: functional tests (windows)
|
||||
permissions: {}
|
||||
on: pull_request
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: RunTestsWindows
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: 'stable'
|
||||
id: go
|
||||
|
||||
- name: Get Dependencies
|
||||
run: go mod download
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
$version = if (Test-Path VERSION) { Get-Content VERSION } else { git describe --abbrev=0 --tags HEAD }
|
||||
$build = git rev-parse --short HEAD
|
||||
$time = (Get-Date -Format "yyyy-MM-dd_hh:mm:sstt")
|
||||
go build -ldflags "-X=main.Build=$build -X=main.BuildTime=$time -X=main.Version=$version" `
|
||||
-o versitygw.exe -cover -race ./cmd/versitygw
|
||||
shell: pwsh
|
||||
|
||||
- name: Run Tests
|
||||
run: pwsh -File .\runtests.ps1
|
||||
shell: pwsh
|
||||
|
||||
- name: Coverage Report
|
||||
run: go tool covdata percent -i="$env:TEMP\covdata"
|
||||
shell: pwsh
|
||||
@@ -10,10 +10,9 @@
|
||||
|
||||
### Binary release builds
|
||||
Download [latest release](https://github.com/versity/versitygw/releases)
|
||||
| Linux/amd64 | Linux/arm64 | MacOS/amd64 | MacOS/arm64 | BSD/amd64 | BSD/arm64 |
|
||||
|:-----------:|:-----------:|:-----------:|:-----------:|:---------:|:---------:|
|
||||
| ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ |
|
||||
|
||||
| Linux amd64/arm64 | MacOS amd64/arm64 | BSD amd64/arm64 | Windows amd64/arm64 |
|
||||
|:-----------:|:-----------:|:-----------:|:-----------:|
|
||||
| ✔️ | ✔️ | ✔️ | ✔️ |
|
||||
### Use Cases
|
||||
* Turn your local filesystem into an S3 server with a single command!
|
||||
* Proxy S3 requests to S3 storage
|
||||
|
||||
@@ -49,6 +49,7 @@ func NewSideCar(dir string) (SideCar, error) {
|
||||
|
||||
// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
|
||||
func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
|
||||
bucket, object = trimVolume(bucket), trimVolume(object)
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
@@ -68,6 +69,7 @@ func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string)
|
||||
|
||||
// StoreAttribute stores the value of a specific attribute for an object or a bucket.
|
||||
func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
|
||||
bucket, object = trimVolume(bucket), trimVolume(object)
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
@@ -114,6 +116,7 @@ func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, va
|
||||
|
||||
// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
|
||||
func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
|
||||
bucket, object = trimVolume(bucket), trimVolume(object)
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
@@ -135,6 +138,7 @@ func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
|
||||
|
||||
// ListAttributes lists all attributes for an object or a bucket.
|
||||
func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
|
||||
bucket, object = trimVolume(bucket), trimVolume(object)
|
||||
metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
|
||||
if object == "" {
|
||||
metadir = filepath.Join(s.dir, bucket, sidecarmeta)
|
||||
@@ -160,6 +164,7 @@ func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
|
||||
// When object is empty the entire bucket sidecar directory is removed,
|
||||
// cleaning up any orphaned object or multipart metadata within it.
|
||||
func (s SideCar) DeleteAttributes(bucket, object string) error {
|
||||
bucket, object = trimVolume(bucket), trimVolume(object)
|
||||
if object == "" {
|
||||
// Remove the entire bucket sidecar directory so that orphaned
|
||||
// object/multipart metadata does not accumulate after DeleteBucket.
|
||||
@@ -184,8 +189,9 @@ func (s SideCar) DeleteAttributes(bucket, object string) error {
|
||||
// newObject so that path-based lookups continue to work after the data
|
||||
// directory has been renamed.
|
||||
func (s SideCar) RenameObject(bucket, oldObject, newObject string) error {
|
||||
oldPath := filepath.Join(s.dir, bucket, oldObject)
|
||||
newPath := filepath.Join(s.dir, bucket, newObject)
|
||||
bucket = trimVolume(bucket)
|
||||
oldPath := filepath.Join(s.dir, bucket, trimVolume(oldObject))
|
||||
newPath := filepath.Join(s.dir, bucket, trimVolume(newObject))
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(newPath), 0777); err != nil {
|
||||
if errors.Is(err, syscall.ENOSPC) {
|
||||
|
||||
@@ -14,12 +14,7 @@
|
||||
|
||||
//go:build !windows
|
||||
|
||||
package posix
|
||||
package meta
|
||||
|
||||
import (
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
func handleParentDirError(_ string) error {
|
||||
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
|
||||
}
|
||||
// trimVolume is a no-op on non-Windows platforms.
|
||||
func trimVolume(p string) string { return p }
|
||||
@@ -14,33 +14,16 @@
|
||||
|
||||
//go:build windows
|
||||
|
||||
package posix
|
||||
package meta
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func handleParentDirError(name string) error {
|
||||
dir := filepath.Dir(name)
|
||||
|
||||
// Walk up the directory hierarchy
|
||||
for dir != "." && dir != "/" {
|
||||
d, statErr := os.Stat(dir)
|
||||
if statErr == nil {
|
||||
// Path component exists
|
||||
if !d.IsDir() {
|
||||
// Found a file in the ancestor path
|
||||
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
|
||||
}
|
||||
// Found a valid directory ancestor, parent truly doesn't exist
|
||||
break
|
||||
}
|
||||
// Continue checking parent directories
|
||||
dir = filepath.Dir(dir)
|
||||
}
|
||||
// Parent doesn't exist or is a directory, treat as ENOENT
|
||||
return nil
|
||||
// trimVolume strips the Windows drive letter (e.g. "C:") and any leading
|
||||
// path separator from p so it can be safely used as a sub-path component
|
||||
// inside filepath.Join without becoming an absolute root.
|
||||
func trimVolume(p string) string {
|
||||
return strings.TrimLeft(p[len(filepath.VolumeName(p)):], `\/`)
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// Copyright 2026 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build !windows
|
||||
|
||||
package posix
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"syscall"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
func handleParentDirError(_ string) error {
|
||||
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
|
||||
}
|
||||
|
||||
// isErrNotDir reports whether err indicates that a path component is a file,
|
||||
// not a directory (POSIX ENOTDIR).
|
||||
func isErrNotDir(err error) bool {
|
||||
return errors.Is(err, syscall.ENOTDIR)
|
||||
}
|
||||
|
||||
// isErrNameTooLong reports whether err indicates that a filename or path
|
||||
// component is too long (POSIX ENAMETOOLONG).
|
||||
func isErrNameTooLong(err error) bool {
|
||||
return errors.Is(err, syscall.ENAMETOOLONG)
|
||||
}
|
||||
|
||||
// isErrDirNotEmpty reports whether err indicates that a directory is not empty
|
||||
// (POSIX ENOTEMPTY).
|
||||
func isErrDirNotEmpty(err error) bool {
|
||||
return errors.Is(err, syscall.ENOTEMPTY)
|
||||
}
|
||||
|
||||
// openForRead opens a file for reading. On non-Windows systems, os.Open is
|
||||
// sufficient because POSIX allows removing (unlinking) a file that is still
|
||||
// open by another process.
|
||||
func openForRead(name string) (*os.File, error) {
|
||||
return os.Open(name)
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
// Copyright 2026 Versity Software
|
||||
// This file is licensed under the Apache License, Version 2.0
|
||||
// (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
//go:build windows
|
||||
|
||||
package posix
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
|
||||
"github.com/versity/versitygw/s3err"
|
||||
)
|
||||
|
||||
func handleParentDirError(name string) error {
|
||||
dir := filepath.Dir(name)
|
||||
|
||||
// Walk up the directory hierarchy
|
||||
for dir != "." && dir != "/" {
|
||||
d, statErr := os.Stat(dir)
|
||||
if statErr == nil {
|
||||
// Path component exists
|
||||
if !d.IsDir() {
|
||||
// Found a file in the ancestor path
|
||||
return s3err.GetAPIError(s3err.ErrObjectParentIsFile)
|
||||
}
|
||||
// Found a valid directory ancestor, parent truly doesn't exist
|
||||
break
|
||||
}
|
||||
// Continue checking parent directories
|
||||
dir = filepath.Dir(dir)
|
||||
}
|
||||
// Parent doesn't exist or is a directory, treat as ENOENT
|
||||
return nil
|
||||
}
|
||||
|
||||
// errDirectory is Windows ERROR_DIRECTORY (267): "The directory name is invalid."
|
||||
// Windows returns this when opening a path like "file/" where "file" is a regular
|
||||
// file rather than a directory — the POSIX equivalent is ENOTDIR.
|
||||
const errDirectory = syscall.Errno(267)
|
||||
|
||||
// errInvalidName is Windows ERROR_INVALID_NAME (123): "The filename, directory
|
||||
// name, or volume label syntax is incorrect." Windows returns this when a path
|
||||
// component exceeds the filesystem name-length limit — the POSIX equivalent is
|
||||
// ENAMETOOLONG.
|
||||
const errInvalidName = syscall.Errno(123)
|
||||
|
||||
// errDirNotEmpty is Windows ERROR_DIR_NOT_EMPTY (145): "The directory is not
|
||||
// empty." — the POSIX equivalent is ENOTEMPTY.
|
||||
const errDirNotEmpty = syscall.Errno(145)
|
||||
|
||||
// isErrNameTooLong reports whether err indicates that a filename or path
|
||||
// component is too long. On Windows this covers both ENAMETOOLONG
|
||||
// (ERROR_FILENAME_EXCED_RANGE, 206) and ERROR_INVALID_NAME (123), which is
|
||||
// what the Windows kernel returns for a 300-character filename that exceeds
|
||||
// MAX_PATH.
|
||||
func isErrNameTooLong(err error) bool {
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
return true
|
||||
}
|
||||
var sysErr syscall.Errno
|
||||
if errors.As(err, &sysErr) {
|
||||
return sysErr == errInvalidName
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isErrDirNotEmpty reports whether err indicates that a directory is not empty.
|
||||
// On Windows this covers both ENOTEMPTY and ERROR_DIR_NOT_EMPTY (145).
|
||||
func isErrDirNotEmpty(err error) bool {
|
||||
if errors.Is(err, syscall.ENOTEMPTY) {
|
||||
return true
|
||||
}
|
||||
var sysErr syscall.Errno
|
||||
if errors.As(err, &sysErr) {
|
||||
return sysErr == errDirNotEmpty
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isErrNotDir reports whether err indicates that a path component is a file,
|
||||
// not a directory. On Windows this covers both ENOTDIR and ERROR_DIRECTORY
|
||||
// because os.Open / os.Stat do not map ERROR_DIRECTORY to ENOTDIR.
|
||||
func isErrNotDir(err error) bool {
|
||||
if errors.Is(err, syscall.ENOTDIR) {
|
||||
return true
|
||||
}
|
||||
var sysErr syscall.Errno
|
||||
if errors.As(err, &sysErr) {
|
||||
return sysErr == errDirectory
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// openForRead opens a file for reading with FILE_SHARE_DELETE so that a
|
||||
// concurrent DeleteObject (os.Remove) can succeed even while the file handle
|
||||
// is held open for streaming the GET response body. Without this flag,
|
||||
// Windows returns "The process cannot access the file because it is being
|
||||
// used by another process" on the Remove call.
|
||||
func openForRead(name string) (*os.File, error) {
|
||||
ptr, err := syscall.UTF16PtrFromString(name)
|
||||
if err != nil {
|
||||
return nil, &os.PathError{Op: "open", Path: name, Err: err}
|
||||
}
|
||||
h, err := syscall.CreateFile(
|
||||
ptr,
|
||||
syscall.GENERIC_READ,
|
||||
syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE,
|
||||
nil,
|
||||
syscall.OPEN_EXISTING,
|
||||
syscall.FILE_ATTRIBUTE_NORMAL,
|
||||
0,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, &os.PathError{Op: "open", Path: name, Err: err}
|
||||
}
|
||||
return os.NewFile(uintptr(h), name), nil
|
||||
}
|
||||
+54
-47
@@ -372,7 +372,7 @@ func (p *Posix) doesBucketAndObjectExist(bucket, object string) error {
|
||||
}
|
||||
|
||||
_, err = os.Stat(filepath.Join(bucket, object))
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -1060,7 +1060,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error {
|
||||
// data file simply doesn't exist — the two cases are indistinguishable
|
||||
// from metadata alone. Verify the data file directly so callers
|
||||
// receive the correct NoSuchVersion / NoSuchKey error.
|
||||
if _, statErr := os.Stat(filepath.Join(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) {
|
||||
if _, statErr := os.Stat(filepath.Join(bucket, object)); errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) {
|
||||
if versionId != "" {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchVersion)
|
||||
}
|
||||
@@ -1068,7 +1068,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error {
|
||||
}
|
||||
|
||||
_, err := p.meta.RetrieveAttribute(nil, bucket, object, deleteMarkerKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if versionId != "" {
|
||||
return s3err.GetNoSuchVersionErr(object, versionId)
|
||||
}
|
||||
@@ -1087,7 +1087,7 @@ func (p *Posix) ensureNotDeleteMarker(bucket, object, versionId string) error {
|
||||
// Check if the given object is a delete marker
|
||||
func (p *Posix) isObjDeleteMarker(bucket, object string) (bool, error) {
|
||||
_, err := p.meta.RetrieveAttribute(nil, bucket, object, deleteMarkerKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return false, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, meta.ErrNoSuchKey) {
|
||||
@@ -3289,7 +3289,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return s3response.CopyPartResult{}, s3err.GetNoSuchUploadErr(*upi.UploadId)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(*upi.Key)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -3360,7 +3360,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
|
||||
}
|
||||
return s3response.CopyPartResult{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return s3response.CopyPartResult{}, s3err.GetKeyTooLongErr(int64(len(srcObject)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -3776,10 +3776,10 @@ func (p *Posix) PutObjectWithPostFunc(ctx context.Context, po s3response.PutObje
|
||||
_ = p.meta.DeleteAttribute(*po.Bucket, *po.Key, objectRetentionKey)
|
||||
}
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return s3response.PutObjectOutput{}, s3err.GetKeyTooLongErr(int64(len(*po.Key)), 1024)
|
||||
}
|
||||
if errors.Is(err, syscall.ENOTDIR) {
|
||||
if isErrNotDir(err) {
|
||||
parentErr := handleParentDirError(name)
|
||||
if parentErr != nil {
|
||||
return s3response.PutObjectOutput{}, parentErr
|
||||
@@ -4067,11 +4067,11 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
if getString(input.VersionId) == "" {
|
||||
// if the versionId is not specified, make the current version a delete marker
|
||||
fi, err := os.Stat(objpath)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
// AWS returns success if the object does not exist
|
||||
return &s3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -4134,7 +4134,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
versionPath := p.genObjVersionPath(bucket, object)
|
||||
|
||||
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
// AWS returns success if the object does not exist
|
||||
return &s3.DeleteObjectOutput{
|
||||
VersionId: input.VersionId,
|
||||
@@ -4151,7 +4151,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
// but "foo" is a regular file (not a directory), the path cannot
|
||||
// contain any object.
|
||||
_, statErr := os.Stat(filepath.Join(bucket, object))
|
||||
if errors.Is(statErr, fs.ErrNotExist) || errors.Is(statErr, syscall.ENOTDIR) {
|
||||
if errors.Is(statErr, fs.ErrNotExist) || isErrNotDir(statErr) {
|
||||
return &s3.DeleteObjectOutput{VersionId: input.VersionId}, nil
|
||||
}
|
||||
vId = []byte(nullVersionId)
|
||||
@@ -4204,7 +4204,6 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open obj version: %w", err)
|
||||
}
|
||||
defer sf.Close()
|
||||
acct, ok := ctx.Value("account").(auth.Account)
|
||||
if !ok {
|
||||
acct = auth.Account{}
|
||||
@@ -4220,9 +4219,14 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
|
||||
_, err = io.Copy(f, sf)
|
||||
if err != nil {
|
||||
_ = sf.Close()
|
||||
return nil, fmt.Errorf("copy object %w", err)
|
||||
}
|
||||
|
||||
if err := sf.Close(); err != nil {
|
||||
return nil, fmt.Errorf("close obj version: %w", err)
|
||||
}
|
||||
|
||||
if err := f.link(); err != nil {
|
||||
return nil, fmt.Errorf("link tmp file: %w", err)
|
||||
}
|
||||
@@ -4275,10 +4279,10 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
isDelMarker, _ := p.isObjDeleteMarker(versionPath, *input.VersionId)
|
||||
|
||||
err = os.Remove(filepath.Join(versionPath, *input.VersionId))
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
|
||||
}
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, *input.VersionId)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -4296,10 +4300,10 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
}
|
||||
|
||||
fi, err := os.Stat(objpath)
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(object)), 1024)
|
||||
}
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
// AWS returns success if the object does not exist
|
||||
return &s3.DeleteObjectOutput{}, nil
|
||||
}
|
||||
@@ -4329,7 +4333,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, syscall.ENOTEMPTY) {
|
||||
if isErrDirNotEmpty(err) {
|
||||
// If the directory object has been uploaded explicitly
|
||||
// remove the directory object (remove the ETag)
|
||||
_, err = p.meta.RetrieveAttribute(nil, objpath, "", etagkey)
|
||||
@@ -4489,7 +4493,7 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
object := *input.Key
|
||||
if versionId != "" {
|
||||
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
@@ -4508,13 +4512,13 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
objPath := filepath.Join(bucket, object)
|
||||
|
||||
fid, err := os.Stat(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if versionId != "" {
|
||||
return nil, s3err.GetNoSuchVersionErr(*input.Key, versionId)
|
||||
}
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -4648,7 +4652,10 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
versionId = string(vId)
|
||||
}
|
||||
|
||||
f, err := os.Open(objPath)
|
||||
// openForRead opens with FILE_SHARE_DELETE on Windows so that a concurrent
|
||||
// DeleteObject can call os.Remove on this file while the GET response body
|
||||
// is still being streamed. On POSIX, os.Open is sufficient.
|
||||
f, err := openForRead(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
@@ -4657,13 +4664,13 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
|
||||
}
|
||||
|
||||
fi, err := f.Stat()
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if versionId != "" {
|
||||
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
|
||||
}
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -4829,7 +4836,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
|
||||
|
||||
if versionId != "" {
|
||||
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
@@ -4849,13 +4856,13 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
|
||||
objPath := filepath.Join(bucket, object)
|
||||
|
||||
fi, err := os.Stat(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if versionId != "" {
|
||||
return nil, s3err.GetNoSuchVersionErr(*input.Key, versionId)
|
||||
}
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetKeyTooLongErr(int64(len(*input.Key)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -5174,7 +5181,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
|
||||
return s3response.CopyObjectOutput{}, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, srcVersionId)
|
||||
}
|
||||
vId, err := p.meta.RetrieveAttribute(nil, srcBucket, srcObject, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
@@ -5197,13 +5204,13 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
|
||||
|
||||
objPath := joinPathWithTrailer(srcBucket, srcObject)
|
||||
f, err := os.Open(objPath)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if p.versioningEnabled() && vEnabled {
|
||||
return s3response.CopyObjectOutput{}, s3err.GetNoSuchVersionErr(origSrcObject, srcVersionId)
|
||||
}
|
||||
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return s3response.CopyObjectOutput{}, s3err.GetKeyTooLongErr(int64(len(origSrcObject)), 1024)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -5344,7 +5351,7 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
|
||||
b, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, etagkey)
|
||||
etag = string(b)
|
||||
vId, _ := p.meta.RetrieveAttribute(nil, dstBucket, dstObject, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return s3response.CopyObjectOutput{}, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
version = backend.GetPtrFromString(string(vId))
|
||||
@@ -5888,10 +5895,10 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
|
||||
|
||||
if versionId == "" {
|
||||
_, err = os.Stat(filepath.Join(bucket, object))
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrKeyTooLong)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -5905,7 +5912,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
|
||||
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
|
||||
}
|
||||
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
@@ -5929,7 +5936,7 @@ func (p *Posix) GetObjectTagging(ctx context.Context, bucket, object, versionId
|
||||
func (p *Posix) getAttrTags(bucket, object, versionId string) (map[string]string, error) {
|
||||
tags := make(map[string]string)
|
||||
b, err := p.meta.RetrieveAttribute(nil, bucket, object, tagHdr)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if versionId != "" {
|
||||
return nil, s3err.GetNoSuchVersionErr(object, versionId)
|
||||
}
|
||||
@@ -5978,10 +5985,10 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
|
||||
|
||||
if versionId == "" {
|
||||
_, err = os.Stat(filepath.Join(bucket, object))
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if errors.Is(err, syscall.ENAMETOOLONG) {
|
||||
if isErrNameTooLong(err) {
|
||||
return s3err.GetAPIError(s3err.ErrKeyTooLong)
|
||||
}
|
||||
if err != nil {
|
||||
@@ -5995,7 +6002,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
|
||||
return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
|
||||
}
|
||||
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
@@ -6015,7 +6022,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
|
||||
|
||||
if tags == nil {
|
||||
err = p.meta.DeleteAttribute(bucket, object, tagHdr)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if versionId != "" {
|
||||
return s3err.GetNoSuchVersionErr(object, versionId)
|
||||
}
|
||||
@@ -6036,7 +6043,7 @@ func (p *Posix) PutObjectTagging(ctx context.Context, bucket, object, versionId
|
||||
}
|
||||
|
||||
err = p.meta.StoreAttribute(nil, bucket, object, tagHdr, b)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if versionId != "" {
|
||||
return s3err.GetNoSuchVersionErr(object, versionId)
|
||||
}
|
||||
@@ -6420,7 +6427,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI
|
||||
return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
|
||||
}
|
||||
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
@@ -6439,7 +6446,7 @@ func (p *Posix) PutObjectLegalHold(ctx context.Context, bucket, object, versionI
|
||||
}
|
||||
|
||||
err = p.meta.StoreAttribute(nil, bucket, object, objectLegalHoldKey, statusData)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if versionId != "" {
|
||||
return s3err.GetNoSuchVersionErr(object, versionId)
|
||||
}
|
||||
@@ -6481,7 +6488,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI
|
||||
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
|
||||
}
|
||||
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
@@ -6500,7 +6507,7 @@ func (p *Posix) GetObjectLegalHold(ctx context.Context, bucket, object, versionI
|
||||
}
|
||||
|
||||
data, err := p.meta.RetrieveAttribute(nil, bucket, object, objectLegalHoldKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if versionId != "" {
|
||||
return nil, s3err.GetNoSuchVersionErr(object, versionId)
|
||||
}
|
||||
@@ -6547,7 +6554,7 @@ func (p *Posix) PutObjectRetention(ctx context.Context, bucket, object, versionI
|
||||
return s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
|
||||
}
|
||||
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
@@ -6602,7 +6609,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI
|
||||
return nil, s3err.GetInvalidArgumentErr(s3err.InvalidArgVersionId, versionId)
|
||||
}
|
||||
vId, err := p.meta.RetrieveAttribute(nil, bucket, object, versionIdKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
return nil, s3err.GetAPIError(s3err.ErrNoSuchKey)
|
||||
}
|
||||
if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
|
||||
@@ -6621,7 +6628,7 @@ func (p *Posix) GetObjectRetention(ctx context.Context, bucket, object, versionI
|
||||
}
|
||||
|
||||
data, err := p.meta.RetrieveAttribute(nil, bucket, object, objectRetentionKey)
|
||||
if errors.Is(err, fs.ErrNotExist) || errors.Is(err, syscall.ENOTDIR) {
|
||||
if errors.Is(err, fs.ErrNotExist) || isErrNotDir(err) {
|
||||
if versionId != "" {
|
||||
return nil, s3err.GetNoSuchVersionErr(object, versionId)
|
||||
}
|
||||
|
||||
@@ -109,7 +109,7 @@ func (tmp *tmpfile) link() error {
|
||||
backoffMs := initialBackoffMs
|
||||
for {
|
||||
err = backend.MoveFile(tempname, objPath, defaultFilePerm)
|
||||
if !errors.Is(err, syscall.ENOENT) {
|
||||
if !os.IsNotExist(err) {
|
||||
break
|
||||
}
|
||||
// The parent directory may have been concurrently removed; backoff and retry.
|
||||
@@ -117,11 +117,12 @@ func (tmp *tmpfile) link() error {
|
||||
sleepWithJitter(backoffMs)
|
||||
backoffMs = min((backoffMs * 2), maxBackoffMs)
|
||||
|
||||
err = backend.MkdirAll(filepath.Dir(objPath), tmp.uid, tmp.gid,
|
||||
// Best-effort: recreate the parent directory. Ignore errors here;
|
||||
// if recreation fails transiently (e.g. Windows pending-delete on
|
||||
// a recently removed directory), the next MoveFile attempt will
|
||||
// return os.IsNotExist again and we will retry.
|
||||
_ = backend.MkdirAll(filepath.Dir(objPath), tmp.uid, tmp.gid,
|
||||
tmp.doChown, tmp.newDirPerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recreate parent dir: %w", err)
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ var (
|
||||
azureTests bool
|
||||
tlsStatus bool
|
||||
parallel bool
|
||||
windowsTests bool
|
||||
sidecarTests bool
|
||||
)
|
||||
|
||||
@@ -121,6 +122,12 @@ func initTestCommands() []*cli.Command {
|
||||
Destination: &azureTests,
|
||||
Aliases: []string{"azure"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "windows-test-mode",
|
||||
Usage: "Skips tests that are not supported on Windows",
|
||||
Destination: &windowsTests,
|
||||
Aliases: []string{"windows"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "sidecar-test-mode",
|
||||
Usage: "Skips tests that are not supported by Sidecar",
|
||||
@@ -176,6 +183,12 @@ func initTestCommands() []*cli.Command {
|
||||
Destination: &versioningEnabled,
|
||||
Aliases: []string{"vs"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "windows-test-mode",
|
||||
Usage: "Skips tests that are not supported on Windows",
|
||||
Destination: &windowsTests,
|
||||
Aliases: []string{"windows"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -421,6 +434,10 @@ func getAction(tf testFunc) func(ctx *cli.Context) error {
|
||||
if azureTests {
|
||||
opts = append(opts, integration.WithAzureMode())
|
||||
}
|
||||
if windowsTests {
|
||||
opts = append(opts, integration.WithWindowsMode())
|
||||
opts = append(opts, integration.WithSidecarMode())
|
||||
}
|
||||
if sidecarTests {
|
||||
opts = append(opts, integration.WithSidecarMode())
|
||||
}
|
||||
@@ -470,6 +487,10 @@ func extractIntTests() (commands []*cli.Command) {
|
||||
if azureTests {
|
||||
opts = append(opts, integration.WithAzureMode())
|
||||
}
|
||||
if windowsTests {
|
||||
opts = append(opts, integration.WithWindowsMode())
|
||||
opts = append(opts, integration.WithSidecarMode())
|
||||
}
|
||||
if sidecarTests {
|
||||
opts = append(opts, integration.WithSidecarMode())
|
||||
}
|
||||
@@ -497,6 +518,12 @@ func extractIntTests() (commands []*cli.Command) {
|
||||
Destination: &sidecarTests,
|
||||
Aliases: []string{"sidecar"},
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "windows-test-mode",
|
||||
Usage: "Skips tests that are not supported on Windows",
|
||||
Destination: &windowsTests,
|
||||
Aliases: []string{"windows"},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env pwsh
|
||||
# Copyright 2026 Versity Software
|
||||
# This file is licensed under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# PowerShell equivalent of runtests.sh for Windows
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
# Temp directories
|
||||
$tmpGw = Join-Path $env:TEMP "gw"
|
||||
$tmpCovdata = Join-Path $env:TEMP "covdata"
|
||||
$tmpHttpsCovdata = Join-Path $env:TEMP "https.covdata"
|
||||
$tmpVersioningCovdata = Join-Path $env:TEMP "versioning.covdata"
|
||||
$tmpVersioningHttpsCovdata = Join-Path $env:TEMP "versioning.https.covdata"
|
||||
$tmpNoaclCovdata = Join-Path $env:TEMP "noacl.covdata"
|
||||
$tmpVersioningDir = Join-Path $env:TEMP "versioningdir"
|
||||
$tmpSidecar = Join-Path $env:TEMP "sidecar"
|
||||
|
||||
foreach ($dir in @($tmpGw, $tmpCovdata, $tmpHttpsCovdata, $tmpVersioningCovdata,
|
||||
$tmpVersioningHttpsCovdata, $tmpNoaclCovdata, $tmpVersioningDir,
|
||||
$tmpSidecar)) {
|
||||
if (Test-Path $dir) { Remove-Item -Recurse -Force $dir }
|
||||
New-Item -ItemType Directory -Path $dir | Out-Null
|
||||
}
|
||||
|
||||
# Setup TLS certificate and key
|
||||
Write-Host "Generating TLS certificate and key in the cert.pem and key.pem files"
|
||||
openssl genpkey -algorithm RSA -out key.pem -pkeyopt rsa_keygen_bits:2048
|
||||
if ($LASTEXITCODE -ne 0) { throw "Failed to generate private key" }
|
||||
openssl req -new -x509 -key key.pem -out cert.pem -days 365 `
|
||||
-subj "/C=US/ST=California/L=San Francisco/O=Versity/OU=Software/CN=versity.com"
|
||||
if ($LASTEXITCODE -ne 0) { throw "Failed to generate certificate" }
|
||||
|
||||
function Start-Gateway {
|
||||
param(
|
||||
[string] $CoverDir,
|
||||
[string[]] $GwArgs
|
||||
)
|
||||
$env:GOCOVERDIR = $CoverDir
|
||||
$proc = Start-Process -FilePath ".\versitygw.exe" -ArgumentList $GwArgs -PassThru -NoNewWindow
|
||||
Remove-Item Env:\GOCOVERDIR -ErrorAction SilentlyContinue
|
||||
return $proc
|
||||
}
|
||||
|
||||
function Invoke-GwTest {
|
||||
param(
|
||||
[string] $Description,
|
||||
[string[]] $TestArgs,
|
||||
[System.Diagnostics.Process] $GatewayProc
|
||||
)
|
||||
& .\versitygw.exe test @TestArgs
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Write-Host "$Description failed"
|
||||
Stop-Process -Id $GatewayProc.Id -Force -ErrorAction SilentlyContinue
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. HTTP (port 7070)
|
||||
# ---------------------------------------------------------------------------
|
||||
Write-Host "Running the sdk test over http"
|
||||
$gwProc = Start-Gateway -CoverDir $tmpCovdata `
|
||||
-GwArgs @("-a", "user", "-s", "pass", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw)
|
||||
Start-Sleep -Seconds 1
|
||||
|
||||
if ($gwProc.HasExited) {
|
||||
Write-Host "server no longer running"
|
||||
exit 1
|
||||
}
|
||||
|
||||
Invoke-GwTest -Description "full flow tests" -GatewayProc $gwProc `
|
||||
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "full-flow", "--parallel", "--windows-test-mode")
|
||||
Invoke-GwTest -Description "posix tests" -GatewayProc $gwProc `
|
||||
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "posix", "--windows-test-mode")
|
||||
Invoke-GwTest -Description "iam tests" -GatewayProc $gwProc `
|
||||
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7070", "iam")
|
||||
|
||||
Stop-Process -Id $gwProc.Id -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. HTTPS (port 7071)
|
||||
# ---------------------------------------------------------------------------
|
||||
Write-Host "Running the sdk test over https"
|
||||
$gwHttpsProc = Start-Gateway -CoverDir $tmpHttpsCovdata `
|
||||
-GwArgs @("--cert", "$PWD\cert.pem", "--key", "$PWD\key.pem",
|
||||
"-p", ":7071", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw)
|
||||
Start-Sleep -Seconds 1
|
||||
|
||||
if ($gwHttpsProc.HasExited) {
|
||||
Write-Host "https server no longer running"
|
||||
exit 1
|
||||
}
|
||||
|
||||
Invoke-GwTest -Description "https full flow tests" -GatewayProc $gwHttpsProc `
|
||||
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "full-flow", "--parallel", "--windows-test-mode")
|
||||
Invoke-GwTest -Description "https posix tests" -GatewayProc $gwHttpsProc `
|
||||
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "posix", "--windows-test-mode")
|
||||
Invoke-GwTest -Description "https iam tests" -GatewayProc $gwHttpsProc `
|
||||
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7071", "iam")
|
||||
|
||||
Stop-Process -Id $gwHttpsProc.Id -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Versioning HTTP (port 7072)
|
||||
# ---------------------------------------------------------------------------
|
||||
Write-Host "Running the sdk test over http against the versioning-enabled gateway"
|
||||
$gwVsProc = Start-Gateway -CoverDir $tmpVersioningCovdata `
|
||||
-GwArgs @("-p", ":7072", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw,
|
||||
"posix", "--sidecar", $tmpSidecar, "--versioning-dir", $tmpVersioningDir, $tmpGw)
|
||||
Start-Sleep -Seconds 1
|
||||
|
||||
if ($gwVsProc.HasExited) {
|
||||
Write-Host "versioning-enabled server no longer running"
|
||||
exit 1
|
||||
}
|
||||
|
||||
Invoke-GwTest -Description "versioning-enabled full-flow tests" -GatewayProc $gwVsProc `
|
||||
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7072", "full-flow", "-vs", "--parallel", "--windows-test-mode")
|
||||
Invoke-GwTest -Description "versioning-enabled posix tests" -GatewayProc $gwVsProc `
|
||||
-TestArgs @("-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7072", "posix", "-vs", "--windows-test-mode")
|
||||
|
||||
Stop-Process -Id $gwVsProc.Id -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 4. Versioning HTTPS (port 7073)
|
||||
# ---------------------------------------------------------------------------
|
||||
Write-Host "Running the sdk test over https against the versioning-enabled gateway"
|
||||
$gwVsHttpsProc = Start-Gateway -CoverDir $tmpVersioningHttpsCovdata `
|
||||
-GwArgs @("--cert", "$PWD\cert.pem", "--key", "$PWD\key.pem",
|
||||
"-p", ":7073", "-a", "user", "-s", "pass", "--iam-dir", $tmpGw,
|
||||
"posix", "--sidecar", $tmpSidecar, "--versioning-dir", $tmpVersioningDir, $tmpGw)
|
||||
Start-Sleep -Seconds 1
|
||||
|
||||
if ($gwVsHttpsProc.HasExited) {
|
||||
Write-Host "versioning-enabled https server no longer running"
|
||||
exit 1
|
||||
}
|
||||
|
||||
Invoke-GwTest -Description "versioning-enabled https full-flow tests" -GatewayProc $gwVsHttpsProc `
|
||||
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7073", "full-flow", "-vs", "--parallel", "--windows-test-mode")
|
||||
Invoke-GwTest -Description "versioning-enabled https posix tests" -GatewayProc $gwVsHttpsProc `
|
||||
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "https://127.0.0.1:7073", "posix", "-vs", "--windows-test-mode")
|
||||
|
||||
Stop-Process -Id $gwVsHttpsProc.Id -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. No ACL (port 7074)
|
||||
# ---------------------------------------------------------------------------
|
||||
Write-Host "Running No ACL integration tests"
|
||||
$gwNoAclProc = Start-Gateway -CoverDir $tmpNoaclCovdata `
|
||||
-GwArgs @("-p", ":7074", "-a", "user", "-s", "pass", "-noacl", "--iam-dir", $tmpGw, "posix", "--sidecar", $tmpSidecar, $tmpGw)
|
||||
Start-Sleep -Seconds 1
|
||||
|
||||
if ($gwNoAclProc.HasExited) {
|
||||
Write-Host "noacl server no longer running"
|
||||
exit 1
|
||||
}
|
||||
|
||||
Invoke-GwTest -Description "No ACL integration tests" -GatewayProc $gwNoAclProc `
|
||||
-TestArgs @("--allow-insecure", "-a", "user", "-s", "pass", "-e", "http://127.0.0.1:7074", "noacl")
|
||||
|
||||
Stop-Process -Id $gwNoAclProc.Id -Force -ErrorAction SilentlyContinue
|
||||
@@ -17,6 +17,7 @@ package integration
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
@@ -90,6 +91,29 @@ func ListBuckets_as_user(s *S3Conf) error {
|
||||
func ListBuckets_as_admin(s *S3Conf) error {
|
||||
testName := "ListBuckets_as_admin"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
testuser, adminUser := getUser("user"), getUser("admin")
|
||||
|
||||
err := createUsers(s, []user{testuser, adminUser})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
adminClient := s.getUserClient(adminUser)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
initOut, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ignore := make([]string, 0, len(initOut.Buckets))
|
||||
for _, b := range initOut.Buckets {
|
||||
if *b.Name != bucket {
|
||||
ignore = append(ignore, *b.Name)
|
||||
}
|
||||
}
|
||||
|
||||
buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
|
||||
for range 6 {
|
||||
bckt := getBucketName()
|
||||
@@ -104,12 +128,6 @@ func ListBuckets_as_admin(s *S3Conf) error {
|
||||
BucketRegion: &s.awsRegion,
|
||||
})
|
||||
}
|
||||
testuser, adminUser := getUser("user"), getUser("admin")
|
||||
|
||||
err := createUsers(s, []user{testuser, adminUser})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bckts := []string{}
|
||||
for i := range 3 {
|
||||
@@ -121,9 +139,7 @@ func ListBuckets_as_admin(s *S3Conf) error {
|
||||
return err
|
||||
}
|
||||
|
||||
adminClient := s.getUserClient(adminUser)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := adminClient.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
cancel()
|
||||
if err != nil {
|
||||
@@ -134,7 +150,7 @@ func ListBuckets_as_admin(s *S3Conf) error {
|
||||
return fmt.Errorf("expected buckets owner to be %v, instead got %v",
|
||||
adminUser.access, getString(out.Owner.ID))
|
||||
}
|
||||
if !compareBuckets(out.Buckets, buckets) {
|
||||
if !compareBuckets(out.Buckets, buckets, ignore...) {
|
||||
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
|
||||
sprintBuckets(buckets), sprintBuckets(out.Buckets))
|
||||
}
|
||||
@@ -154,6 +170,21 @@ func ListBuckets_with_prefix(s *S3Conf) error {
|
||||
testName := "ListBuckets_with_prefix"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
prefix := "my-prefix-"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
prefixInitOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
|
||||
Prefix: &prefix,
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ignore := make([]string, 0, len(prefixInitOut.Buckets))
|
||||
for _, b := range prefixInitOut.Buckets {
|
||||
ignore = append(ignore, *b.Name)
|
||||
}
|
||||
|
||||
allBuckets, prefixedBuckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}, []types.Bucket{}
|
||||
for i := range 5 {
|
||||
bckt := getBucketName()
|
||||
@@ -179,7 +210,7 @@ func ListBuckets_with_prefix(s *S3Conf) error {
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
|
||||
Prefix: &prefix,
|
||||
})
|
||||
@@ -196,7 +227,7 @@ func ListBuckets_with_prefix(s *S3Conf) error {
|
||||
return fmt.Errorf("expected prefix to be %v, instead got %v",
|
||||
prefix, getString(out.Prefix))
|
||||
}
|
||||
if !compareBuckets(out.Buckets, prefixedBuckets) {
|
||||
if !compareBuckets(out.Buckets, prefixedBuckets, ignore...) {
|
||||
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
|
||||
prefixedBuckets, out.Buckets)
|
||||
}
|
||||
@@ -243,7 +274,14 @@ func ListBuckets_invalid_max_buckets(s *S3Conf) error {
|
||||
func ListBuckets_truncated(s *S3Conf) error {
|
||||
testName := "ListBuckets_truncated"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newBuckets := []types.Bucket{}
|
||||
for range 5 {
|
||||
bckt := getBucketName()
|
||||
|
||||
@@ -252,14 +290,19 @@ func ListBuckets_truncated(s *S3Conf) error {
|
||||
return err
|
||||
}
|
||||
|
||||
buckets = append(buckets, types.Bucket{
|
||||
newBuckets = append(newBuckets, types.Bucket{
|
||||
Name: &bckt,
|
||||
BucketRegion: &s.awsRegion,
|
||||
})
|
||||
}
|
||||
|
||||
buckets := append(initOut.Buckets, newBuckets...)
|
||||
sort.Slice(buckets, func(i, j int) bool {
|
||||
return *buckets[i].Name < *buckets[j].Name
|
||||
})
|
||||
|
||||
maxBuckets := int32(3)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{
|
||||
MaxBuckets: &maxBuckets,
|
||||
})
|
||||
@@ -292,7 +335,7 @@ func ListBuckets_truncated(s *S3Conf) error {
|
||||
|
||||
if !compareBuckets(out.Buckets, buckets[maxBuckets:]) {
|
||||
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
|
||||
sprintBuckets(buckets[:maxBuckets]), sprintBuckets(out.Buckets))
|
||||
sprintBuckets(buckets[maxBuckets:]), sprintBuckets(out.Buckets))
|
||||
}
|
||||
if out.ContinuationToken != nil {
|
||||
return fmt.Errorf("expected nil continuation token, instead got %v",
|
||||
@@ -302,7 +345,7 @@ func ListBuckets_truncated(s *S3Conf) error {
|
||||
return fmt.Errorf("expected nil prefix, instead got %v", *out.Prefix)
|
||||
}
|
||||
|
||||
for _, elem := range buckets[1:] {
|
||||
for _, elem := range newBuckets {
|
||||
err = teardown(s, *elem.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -335,6 +378,20 @@ func ListBuckets_empty_success(s *S3Conf) error {
|
||||
func ListBuckets_success(s *S3Conf) error {
|
||||
testName := "ListBuckets_success"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
initOut, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ignore := make([]string, 0, len(initOut.Buckets))
|
||||
for _, b := range initOut.Buckets {
|
||||
if *b.Name != bucket {
|
||||
ignore = append(ignore, *b.Name)
|
||||
}
|
||||
}
|
||||
|
||||
buckets := []types.Bucket{{Name: &bucket, BucketRegion: &s.awsRegion}}
|
||||
for range 5 {
|
||||
bckt := getBucketName()
|
||||
@@ -350,7 +407,7 @@ func ListBuckets_success(s *S3Conf) error {
|
||||
})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
ctx, cancel = context.WithTimeout(context.Background(), shortTimeout)
|
||||
out, err := s3client.ListBuckets(ctx, &s3.ListBucketsInput{})
|
||||
cancel()
|
||||
if err != nil {
|
||||
@@ -361,7 +418,7 @@ func ListBuckets_success(s *S3Conf) error {
|
||||
return fmt.Errorf("expected owner to be %v, instead got %v",
|
||||
s.awsID, getString(out.Owner.ID))
|
||||
}
|
||||
if !compareBuckets(out.Buckets, buckets) {
|
||||
if !compareBuckets(out.Buckets, buckets, ignore...) {
|
||||
return fmt.Errorf("expected list buckets result to be %v, instead got %v",
|
||||
sprintBuckets(buckets), sprintBuckets(out.Buckets))
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -52,10 +53,21 @@ func PutObject_special_chars(s *S3Conf) error {
|
||||
"my?key", "my^key", "my{}key", "my%key", "my`key",
|
||||
"my[]key", "my~key", "my<>key", "my|key", "my#key",
|
||||
}
|
||||
if !s.azureTests {
|
||||
// azure currently can't handle backslashes in object names
|
||||
if !s.azureTests && !s.windowsTests {
|
||||
// azure and windows cannot handle backslashes in object names:
|
||||
// on Windows, '\' is a path separator so 'my\key' is stored as 'my/key'
|
||||
objnames = append(objnames, "my\\key")
|
||||
}
|
||||
if s.windowsTests {
|
||||
// ':', '?', '<', '>', '|', '*' are not valid filename characters on Windows
|
||||
var filtered []string
|
||||
for _, name := range objnames {
|
||||
if !strings.ContainsAny(name, `:?<>|*`) {
|
||||
filtered = append(filtered, name)
|
||||
}
|
||||
}
|
||||
objnames = filtered
|
||||
}
|
||||
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
objs, err := putObjects(s3client, objnames, bucket)
|
||||
@@ -1169,19 +1181,30 @@ func PutObject_false_negative_object_names(s *S3Conf) error {
|
||||
testName := "PutObject_false_negative_object_names"
|
||||
return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error {
|
||||
objs := []string{
|
||||
"%252e%252e%252fetc/passwd", // double encoding
|
||||
"%2e%2e/%2e%2e/%2e%2e/.ssh/id_rsa", // double URL-encoded
|
||||
"%u002e%u002e/%u002e%u002e/etc/passwd", // unicode escape
|
||||
"..%2f..%2f..%2fsecret/file.txt", // URL-encoded
|
||||
"..%c0%af..%c0%afetc/passwd", // UTF-8 overlong trick
|
||||
".../.../.../target.txt",
|
||||
"..\\u2215..\\u2215etc/passwd", // Unicode division slash
|
||||
"dir/%20../file.txt", // encoded space
|
||||
"%252e%252e%252fetc/passwd", // double encoding
|
||||
"%2e%2e/%2e%2e/%2e%2e/.ssh/id_rsa", // double URL-encoded
|
||||
"%u002e%u002e/%u002e%u002e/etc/passwd", // unicode escape
|
||||
"..%2f..%2f..%2fsecret/file.txt", // URL-encoded
|
||||
"..%c0%af..%c0%afetc/passwd", // UTF-8 overlong trick
|
||||
"dir/%c0%ae%c0%ae/%c0%ae%c0%ae/etc/passwd", // overlong UTF-8 encoding
|
||||
"logs/latest -> /etc/passwd", // symlink attacks
|
||||
//TODO: add this test case in advanced routing
|
||||
// "/etc/passwd" // absolute path injection
|
||||
}
|
||||
if !s.windowsTests {
|
||||
// Windows strips trailing dots from path components, making '...' an
|
||||
// invalid directory name (the filesystem rejects MkdirAll with it).
|
||||
objs = append(objs, ".../.../.../target.txt")
|
||||
objs = append(objs, "dir/%20../file.txt") // encoded space
|
||||
// literal backslashes are treated as path separators on Windows
|
||||
objs = append(objs, "..\\u2215..\\u2215etc/passwd") // Unicode division slash
|
||||
// On Windows, '>' is an invalid filename character. The key
|
||||
// 'logs/latest -> /etc/passwd' creates a directory component
|
||||
// 'latest -> ' (containing '>') which MkdirAll rejects.
|
||||
objs = append(objs, "logs/latest -> /etc/passwd") // symlink attacks
|
||||
}
|
||||
|
||||
sort.Strings(objs)
|
||||
|
||||
_, err := putObjects(s3client, objs, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -196,7 +196,7 @@ func TestPutObject(ts *TestState) {
|
||||
}
|
||||
ts.Run(PutObject_success)
|
||||
ts.Run(PutObject_default_content_type)
|
||||
if !ts.conf.versioningEnabled {
|
||||
if !ts.conf.versioningEnabled && !ts.conf.windowsTests {
|
||||
ts.Run(PutObject_racey_success)
|
||||
}
|
||||
ts.Run(PutObject_invalid_credentials)
|
||||
@@ -957,7 +957,9 @@ func TestPosix(ts *TestState) {
|
||||
ts.Run(CopyObject_overwrite_same_dir_object)
|
||||
ts.Run(CopyObject_overwrite_same_file_object)
|
||||
ts.Run(DeleteObject_directory_not_empty)
|
||||
ts.Run(PutObject_race_with_delete)
|
||||
if !ts.conf.windowsTests {
|
||||
ts.Run(PutObject_race_with_delete)
|
||||
}
|
||||
// posix specific versioning tests
|
||||
if !ts.conf.versioningEnabled {
|
||||
TestVersioningDisabled(ts)
|
||||
@@ -1138,10 +1140,14 @@ func TestVersioning(ts *TestState) {
|
||||
ts.Run(Versioning_CopyObject_success)
|
||||
ts.Run(Versioning_CopyObject_non_existing_version_id)
|
||||
ts.Run(Versioning_CopyObject_from_an_object_version)
|
||||
ts.Run(Versioning_CopyObject_special_chars)
|
||||
if !ts.conf.windowsTests {
|
||||
ts.Run(Versioning_CopyObject_special_chars)
|
||||
}
|
||||
// HeadObject action
|
||||
ts.Run(Versioning_HeadObject_invalid_versionId)
|
||||
ts.Run(Versioning_HeadObject_non_existing_object_version)
|
||||
if !ts.conf.windowsTests {
|
||||
ts.Run(Versioning_HeadObject_non_existing_object_version)
|
||||
}
|
||||
ts.Run(Versioning_HeadObject_invalid_parent)
|
||||
ts.Run(Versioning_HeadObject_success)
|
||||
ts.Run(Versioning_HeadObject_without_versionId)
|
||||
@@ -1170,7 +1176,9 @@ func TestVersioning(ts *TestState) {
|
||||
ts.Run(Versioning_DeleteObject_invalid_versionId)
|
||||
ts.Run(Versioning_DeleteObject_delete_object_version)
|
||||
ts.Run(Versioning_DeleteObject_non_existing_object)
|
||||
ts.Run(Versioning_DeleteObject_delete_a_delete_marker)
|
||||
if !ts.conf.windowsTests {
|
||||
ts.Run(Versioning_DeleteObject_delete_a_delete_marker)
|
||||
}
|
||||
ts.Run(Versioning_Delete_null_versionId_object)
|
||||
ts.Run(Versioning_DeleteObject_nested_dir_object)
|
||||
ts.Run(Versioning_DeleteObject_non_existing_objects)
|
||||
@@ -1221,13 +1229,17 @@ func TestVersioning(ts *TestState) {
|
||||
ts.Run(Versioning_WORM_PutObject_overwrite_locked_object)
|
||||
ts.Run(Versioning_WORM_CopyObject_overwrite_locked_object)
|
||||
ts.Run(Versioning_WORM_CompleteMultipartUpload_overwrite_locked_object)
|
||||
ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention)
|
||||
if !ts.conf.windowsTests {
|
||||
ts.Run(Versioning_WORM_remove_delete_marker_under_bucket_default_retention)
|
||||
}
|
||||
// Concurrent requests
|
||||
// Versioninig_concurrent_upload_object
|
||||
ts.Run(Versioning_AccessControl_GetObjectVersion)
|
||||
ts.Run(Versioning_AccessControl_HeadObjectVersion)
|
||||
ts.Run(Versioning_AccessControl_object_tagging_policy)
|
||||
ts.Run(Versioning_AccessControl_DeleteObject_policy)
|
||||
if !ts.conf.windowsTests {
|
||||
ts.Run(Versioning_AccessControl_DeleteObject_policy)
|
||||
}
|
||||
ts.Run(Versioning_AccessControl_GetObjectAttributes_policy)
|
||||
}
|
||||
|
||||
|
||||
@@ -852,6 +852,10 @@ func PresignedAuth_Put_GetObject_with_UTF8_chars(s *S3Conf) error {
|
||||
testName := "PresignedAuth_Put_GetObject_with_UTF8_chars"
|
||||
return presignedAuthHandler(s, testName, func(client *s3.PresignClient, bucket string) error {
|
||||
obj := "my-$%^&*;"
|
||||
if s.windowsTests {
|
||||
// '*' is not a valid filename character on Windows
|
||||
obj = "my-$%^&;"
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
v4req, err := client.PresignPutObject(ctx, &s3.PutObjectInput{Bucket: &bucket, Key: &obj})
|
||||
|
||||
@@ -46,6 +46,7 @@ type S3Conf struct {
|
||||
debug bool
|
||||
versioningEnabled bool
|
||||
azureTests bool
|
||||
windowsTests bool
|
||||
sidecarTests bool
|
||||
tlsStatus bool
|
||||
httpClient *http.Client
|
||||
@@ -121,6 +122,9 @@ func WithVersioningEnabled() Option {
|
||||
func WithAzureMode() Option {
|
||||
return func(s *S3Conf) { s.azureTests = true }
|
||||
}
|
||||
func WithWindowsMode() Option {
|
||||
return func(s *S3Conf) { s.windowsTests = true }
|
||||
}
|
||||
func WithSidecarMode() Option {
|
||||
return func(s *S3Conf) { s.sidecarTests = true }
|
||||
}
|
||||
|
||||
@@ -1502,7 +1502,21 @@ func areMapsSame(mp1, mp2 map[string]string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func compareBuckets(list1 []types.Bucket, list2 []types.Bucket) bool {
|
||||
func compareBuckets(list1 []types.Bucket, list2 []types.Bucket, ignore ...string) bool {
|
||||
if len(ignore) > 0 {
|
||||
ignoreSet := make(map[string]struct{}, len(ignore))
|
||||
for _, name := range ignore {
|
||||
ignoreSet[name] = struct{}{}
|
||||
}
|
||||
filtered := make([]types.Bucket, 0, len(list1))
|
||||
for _, b := range list1 {
|
||||
if _, skip := ignoreSet[*b.Name]; !skip {
|
||||
filtered = append(filtered, b)
|
||||
}
|
||||
}
|
||||
list1 = filtered
|
||||
}
|
||||
|
||||
if len(list1) != len(list2) {
|
||||
return false
|
||||
}
|
||||
@@ -1771,8 +1785,17 @@ func listBuckets(s *S3Conf) error {
|
||||
|
||||
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
|
||||
// randCounter is an atomic counter used by genRandString. It is seeded once
|
||||
// with time.Now().UnixNano() so that values are unique even when many goroutines
|
||||
// call genRandString concurrently on systems with low-resolution clocks.
|
||||
var randCounter = atomic.Uint64{}
|
||||
|
||||
func init() {
|
||||
randCounter.Store(uint64(time.Now().UnixNano()))
|
||||
}
|
||||
|
||||
func genRandString(length int) string {
|
||||
source := rnd.NewSource(time.Now().UnixNano())
|
||||
source := rnd.NewSource(int64(randCounter.Add(1)))
|
||||
random := rnd.New(source)
|
||||
result := make([]byte, length)
|
||||
for i := range result {
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -437,12 +438,17 @@ func Versioning_CopyObject_special_chars(s *S3Conf) error {
|
||||
}
|
||||
|
||||
srcObjVersionId := *srcObjVersions[0].VersionId
|
||||
copySource := fmt.Sprintf("%v/%v?versionId=%v",
|
||||
bucket,
|
||||
url.PathEscape(srcObj),
|
||||
url.QueryEscape(srcObjVersionId),
|
||||
)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), shortTimeout)
|
||||
res, err := s3client.CopyObject(ctx, &s3.CopyObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &dstObj,
|
||||
CopySource: getPtr(fmt.Sprintf("%v/%v?versionId=%v", bucket, srcObj, srcObjVersionId)),
|
||||
CopySource: getPtr(copySource),
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user