mirror of
https://github.com/versity/versitygw.git
synced 2025-12-23 05:05:16 +00:00
379 lines
8.3 KiB
Go
379 lines
8.3 KiB
Go
// Copyright 2023 Versity Software
|
|
// This file is licensed under the Apache License, Version 2.0
|
|
// (the "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing,
|
|
// software distributed under the License is distributed on an
|
|
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
// KIND, either express or implied. See the License for the
|
|
// specific language governing permissions and limitations
|
|
// under the License.
|
|
|
|
package backend_test
|
|
|
|
import (
|
|
"context"
|
|
"crypto/md5"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io/fs"
|
|
"sync"
|
|
"testing"
|
|
"testing/fstest"
|
|
"time"
|
|
|
|
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
|
"github.com/versity/versitygw/backend"
|
|
"github.com/versity/versitygw/s3response"
|
|
)
|
|
|
|
type walkTest struct {
|
|
fsys fs.FS
|
|
getobj backend.GetObjFunc
|
|
cases []testcase
|
|
}
|
|
|
|
type testcase struct {
|
|
name string
|
|
prefix string
|
|
delimiter string
|
|
marker string
|
|
maxObjs int32
|
|
expected backend.WalkResults
|
|
}
|
|
|
|
func getObj(path string, d fs.DirEntry) (s3response.Object, error) {
|
|
if d.IsDir() {
|
|
etag := getMD5(path)
|
|
|
|
fi, err := d.Info()
|
|
if err != nil {
|
|
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
|
}
|
|
mtime := fi.ModTime()
|
|
|
|
return s3response.Object{
|
|
ETag: &etag,
|
|
Key: &path,
|
|
LastModified: &mtime,
|
|
}, nil
|
|
}
|
|
|
|
etag := getMD5(path)
|
|
|
|
fi, err := d.Info()
|
|
if err != nil {
|
|
return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err)
|
|
}
|
|
|
|
size := fi.Size()
|
|
mtime := fi.ModTime()
|
|
|
|
return s3response.Object{
|
|
ETag: &etag,
|
|
Key: &path,
|
|
LastModified: &mtime,
|
|
Size: &size,
|
|
}, nil
|
|
}
|
|
|
|
func getMD5(text string) string {
|
|
hash := md5.Sum([]byte(text))
|
|
return hex.EncodeToString(hash[:])
|
|
}
|
|
|
|
func TestWalk(t *testing.T) {
|
|
tests := []walkTest{
|
|
{
|
|
// test case from
|
|
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-prefixes.html
|
|
fsys: fstest.MapFS{
|
|
"sample.jpg": {},
|
|
"photos/2006/January/sample.jpg": {},
|
|
"photos/2006/February/sample2.jpg": {},
|
|
"photos/2006/February/sample3.jpg": {},
|
|
"photos/2006/February/sample4.jpg": {},
|
|
},
|
|
getobj: getObj,
|
|
cases: []testcase{
|
|
{
|
|
name: "aws example",
|
|
delimiter: "/",
|
|
maxObjs: 1000,
|
|
expected: backend.WalkResults{
|
|
CommonPrefixes: []types.CommonPrefix{{
|
|
Prefix: backend.GetPtrFromString("photos/"),
|
|
}},
|
|
Objects: []s3response.Object{{
|
|
Key: backend.GetPtrFromString("sample.jpg"),
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
// test case single dir/single file
|
|
fsys: fstest.MapFS{
|
|
"test/file": {},
|
|
},
|
|
getobj: getObj,
|
|
cases: []testcase{
|
|
{
|
|
name: "single dir single file",
|
|
delimiter: "/",
|
|
maxObjs: 1000,
|
|
expected: backend.WalkResults{
|
|
CommonPrefixes: []types.CommonPrefix{{
|
|
Prefix: backend.GetPtrFromString("test/"),
|
|
}},
|
|
Objects: []s3response.Object{},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
// non-standard delimiter
|
|
fsys: fstest.MapFS{
|
|
"photo|s/200|6/Januar|y/sampl|e1.jpg": {},
|
|
"photo|s/200|6/Januar|y/sampl|e2.jpg": {},
|
|
"photo|s/200|6/Januar|y/sampl|e3.jpg": {},
|
|
},
|
|
getobj: getObj,
|
|
cases: []testcase{
|
|
{
|
|
name: "different delimiter 1",
|
|
delimiter: "|",
|
|
maxObjs: 1000,
|
|
expected: backend.WalkResults{
|
|
CommonPrefixes: []types.CommonPrefix{{
|
|
Prefix: backend.GetPtrFromString("photo|"),
|
|
}},
|
|
},
|
|
},
|
|
{
|
|
name: "different delimiter 2",
|
|
delimiter: "|",
|
|
maxObjs: 1000,
|
|
prefix: "photo|",
|
|
expected: backend.WalkResults{
|
|
CommonPrefixes: []types.CommonPrefix{{
|
|
Prefix: backend.GetPtrFromString("photo|s/200|"),
|
|
}},
|
|
},
|
|
},
|
|
{
|
|
name: "different delimiter 3",
|
|
delimiter: "|",
|
|
maxObjs: 1000,
|
|
prefix: "photo|s/200|",
|
|
expected: backend.WalkResults{
|
|
CommonPrefixes: []types.CommonPrefix{{
|
|
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
|
|
}},
|
|
},
|
|
},
|
|
{
|
|
name: "different delimiter 4",
|
|
delimiter: "|",
|
|
maxObjs: 1000,
|
|
prefix: "photo|s/200|",
|
|
expected: backend.WalkResults{
|
|
CommonPrefixes: []types.CommonPrefix{{
|
|
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|"),
|
|
}},
|
|
},
|
|
},
|
|
{
|
|
name: "different delimiter 5",
|
|
delimiter: "|",
|
|
maxObjs: 1000,
|
|
prefix: "photo|s/200|6/Januar|",
|
|
expected: backend.WalkResults{
|
|
CommonPrefixes: []types.CommonPrefix{{
|
|
Prefix: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|"),
|
|
}},
|
|
},
|
|
},
|
|
{
|
|
name: "different delimiter 6",
|
|
delimiter: "|",
|
|
maxObjs: 1000,
|
|
prefix: "photo|s/200|6/Januar|y/sampl|",
|
|
expected: backend.WalkResults{
|
|
Objects: []s3response.Object{
|
|
{
|
|
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e1.jpg"),
|
|
},
|
|
{
|
|
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e2.jpg"),
|
|
},
|
|
{
|
|
Key: backend.GetPtrFromString("photo|s/200|6/Januar|y/sampl|e3.jpg"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
for _, tc := range tt.cases {
|
|
res, err := backend.Walk(context.Background(),
|
|
tt.fsys, tc.prefix, tc.delimiter, tc.marker, tc.maxObjs,
|
|
tt.getobj, []string{})
|
|
if err != nil {
|
|
t.Errorf("tc.name: walk: %v", err)
|
|
}
|
|
|
|
compareResults(tc.name, res, tc.expected, t)
|
|
}
|
|
}
|
|
}
|
|
|
|
func compareResults(name string, got, wanted backend.WalkResults, t *testing.T) {
|
|
if !compareCommonPrefix(got.CommonPrefixes, wanted.CommonPrefixes) {
|
|
t.Errorf("%v: unexpected common prefix, got %v wanted %v",
|
|
name,
|
|
printCommonPrefixes(got.CommonPrefixes),
|
|
printCommonPrefixes(wanted.CommonPrefixes))
|
|
}
|
|
|
|
if !compareObjects(got.Objects, wanted.Objects) {
|
|
t.Errorf("%v: unexpected object, got %v wanted %v",
|
|
name,
|
|
printObjects(got.Objects),
|
|
printObjects(wanted.Objects))
|
|
}
|
|
}
|
|
|
|
func compareCommonPrefix(a, b []types.CommonPrefix) bool {
|
|
if len(a) == 0 && len(b) == 0 {
|
|
return true
|
|
}
|
|
if len(a) != len(b) {
|
|
return false
|
|
}
|
|
|
|
for _, cp := range a {
|
|
if containsCommonPrefix(cp, b) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func containsCommonPrefix(c types.CommonPrefix, list []types.CommonPrefix) bool {
|
|
for _, cp := range list {
|
|
if *c.Prefix == *cp.Prefix {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func printCommonPrefixes(list []types.CommonPrefix) string {
|
|
res := "["
|
|
for _, cp := range list {
|
|
if res == "[" {
|
|
res = res + *cp.Prefix
|
|
} else {
|
|
res = res + ", " + *cp.Prefix
|
|
}
|
|
}
|
|
return res + "]"
|
|
}
|
|
|
|
func compareObjects(a, b []s3response.Object) bool {
|
|
if len(a) == 0 && len(b) == 0 {
|
|
return true
|
|
}
|
|
if len(a) != len(b) {
|
|
return false
|
|
}
|
|
|
|
for _, cp := range a {
|
|
if containsObject(cp, b) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func containsObject(c s3response.Object, list []s3response.Object) bool {
|
|
for _, cp := range list {
|
|
if *c.Key == *cp.Key {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func printObjects(list []s3response.Object) string {
|
|
res := "["
|
|
for _, cp := range list {
|
|
var key string
|
|
if cp.Key == nil {
|
|
key = "<nil>"
|
|
} else {
|
|
key = *cp.Key
|
|
}
|
|
if res == "[" {
|
|
res = res + key
|
|
} else {
|
|
res = res + ", " + key
|
|
}
|
|
}
|
|
return res + "]"
|
|
}
|
|
|
|
type slowFS struct {
|
|
fstest.MapFS
|
|
}
|
|
|
|
const (
|
|
readDirPause = 100 * time.Millisecond
|
|
|
|
// walkTimeOut should be less than the tree traversal time
|
|
// which is the readdirPause time * the number of directories
|
|
walkTimeOut = 500 * time.Millisecond
|
|
)
|
|
|
|
func (s *slowFS) ReadDir(name string) ([]fs.DirEntry, error) {
|
|
time.Sleep(readDirPause)
|
|
return s.MapFS.ReadDir(name)
|
|
}
|
|
|
|
func TestWalkStop(t *testing.T) {
|
|
s := &slowFS{MapFS: fstest.MapFS{
|
|
"/a/b/c/d/e/f/g/h/i/g/k/l/m/n": &fstest.MapFile{},
|
|
}}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), walkTimeOut)
|
|
defer cancel()
|
|
|
|
var err error
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
_, err = backend.Walk(ctx, s, "", "/", "", 1000,
|
|
func(path string, d fs.DirEntry) (s3response.Object, error) {
|
|
return s3response.Object{}, nil
|
|
}, []string{})
|
|
}()
|
|
|
|
select {
|
|
case <-time.After(1 * time.Second):
|
|
t.Fatalf("walk is not terminated in time")
|
|
case <-ctx.Done():
|
|
}
|
|
wg.Wait()
|
|
if err != ctx.Err() {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
}
|