diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 1834584..6245c7a 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1031,19 +1031,7 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) ( fileSystem := os.DirFS(bucket) results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys, - func(path string) (bool, error) { - _, err := xattr.Get(filepath.Join(bucket, path), etagkey) - if isNoAttr(err) { - return false, nil - } - if err != nil { - return false, err - } - return true, nil - }, func(path string) (string, error) { - etag, err := xattr.Get(filepath.Join(bucket, path), etagkey) - return string(etag), err - }, []string{metaTmpDir}) + fileToObj(bucket), []string{metaTmpDir}) if err != nil { return nil, fmt.Errorf("walk %v: %w", bucket, err) } @@ -1061,6 +1049,65 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) ( }, nil } +func fileToObj(bucket string) backend.GetObjFunc { + return func(path string, d fs.DirEntry) (types.Object, error) { + if d.IsDir() { + // directory object only happens if directory empty + // check to see if this is a directory object by checking etag + etagBytes, err := xattr.Get(filepath.Join(bucket, path), etagkey) + if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) { + return types.Object{}, backend.ErrSkipObj + } + if err != nil { + return types.Object{}, fmt.Errorf("get etag: %w", err) + } + etag := string(etagBytes) + + fi, err := d.Info() + if errors.Is(err, fs.ErrNotExist) { + return types.Object{}, backend.ErrSkipObj + } + if err != nil { + return types.Object{}, fmt.Errorf("get fileinfo: %w", err) + } + + return types.Object{ + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(fi.ModTime()), + }, nil + } + + // file object, get object info and fill out object data + etagBytes, err := xattr.Get(filepath.Join(bucket, path), etagkey) + if errors.Is(err, fs.ErrNotExist) { + return types.Object{}, backend.ErrSkipObj + } + if err != nil && !isNoAttr(err) { + return types.Object{}, fmt.Errorf("get etag: %w", err) + } + // note: isNoAttr(err) will return etagBytes = []byte{} + // so this will just set etag to "" if its not already set + + etag := string(etagBytes) + + fi, err := d.Info() + if errors.Is(err, fs.ErrNotExist) { + return types.Object{}, backend.ErrSkipObj + } + if err != nil { + return types.Object{}, fmt.Errorf("get fileinfo: %w", err) + } + + return types.Object{ + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(fi.ModTime()), + Size: fi.Size(), + }, nil + } +} + func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) { _, err := os.Stat(bucket) if errors.Is(err, fs.ErrNotExist) { @@ -1072,19 +1119,7 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) fileSystem := os.DirFS(bucket) results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys, - func(path string) (bool, error) { - _, err := xattr.Get(filepath.Join(bucket, path), etagkey) - if isNoAttr(err) { - return false, nil - } - if err != nil { - return false, err - } - return true, nil - }, func(path string) (string, error) { - etag, err := xattr.Get(filepath.Join(bucket, path), etagkey) - return string(etag), err - }, []string{metaTmpDir}) + fileToObj(bucket), []string{metaTmpDir}) if err != nil { return nil, fmt.Errorf("walk %v: %w", bucket, err) } diff --git a/backend/walk.go b/backend/walk.go index c66d967..0d31277 100644 --- a/backend/walk.go +++ b/backend/walk.go @@ -15,6 +15,7 @@ package backend import ( + "errors" "fmt" "io/fs" "os" @@ -31,12 +32,13 @@ type WalkResults struct { NextMarker string } -type DirObjCheck func(path string) (bool, error) -type GetETag func(path string) (string, error) +type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error) + +var ErrSkipObj = errors.New("skip this object") // Walk walks the supplied fs.FS and returns results compatible with list // objects responses -func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, dirchk DirObjCheck, getetag GetETag, skipdirs []string) (WalkResults, error) { +func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj GetObjFunc, skipdirs []string) (WalkResults, error) { cpmap := make(map[string]struct{}) var objects []types.Object @@ -90,26 +92,14 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, dirchk Di return fmt.Errorf("readdir %q: %w", path, err) } if len(ents) == 0 { - dirobj, err := dirchk(path) + dirobj, err := getObj(path, d) + if err == ErrSkipObj { + return nil + } if err != nil { - return fmt.Errorf("directory object check %q: %w", path, err) - } - if dirobj { - fi, err := d.Info() - if err != nil { - return fmt.Errorf("dir info %q: %w", path, err) - } - etag, err := getetag(path) - if err != nil { - return fmt.Errorf("get etag %q: %w", path, err) - } - path := path + "/" - objects = append(objects, types.Object{ - ETag: &etag, - Key: &path, - LastModified: GetTimePtr(fi.ModTime()), - }) + return fmt.Errorf("directory to object %q: %w", path, err) } + objects = append(objects, dirobj) } return nil @@ -130,21 +120,14 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, dirchk Di if delimiter == "" { // If no delimeter specified, then all files with matching // prefix are included in results - fi, err := d.Info() - if err != nil { - return fmt.Errorf("get info for %v: %w", path, err) + obj, err := getObj(path, d) + if err == ErrSkipObj { + return nil } - etag, err := getetag(path) if err != nil { - return fmt.Errorf("get etag %q: %w", path, err) + return fmt.Errorf("file to object %q: %w", path, err) } - - objects = append(objects, types.Object{ - ETag: &etag, - Key: &path, - LastModified: GetTimePtr(fi.ModTime()), - Size: fi.Size(), - }) + objects = append(objects, obj) if max > 0 && (len(objects)+len(cpmap)) == max { pastMax = true @@ -177,20 +160,14 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, dirchk Di suffix := strings.TrimPrefix(path, prefix) before, _, found := strings.Cut(suffix, delimiter) if !found { - fi, err := d.Info() - if err != nil { - return fmt.Errorf("get info for %v: %w", path, err) + obj, err := getObj(path, d) + if err == ErrSkipObj { + return nil } - etag, err := getetag(path) if err != nil { - return fmt.Errorf("get etag %q: %w", path, err) + return fmt.Errorf("file to object %q: %w", path, err) } - objects = append(objects, types.Object{ - ETag: &etag, - Key: &path, - LastModified: GetTimePtr(fi.ModTime()), - Size: fi.Size(), - }) + objects = append(objects, obj) if (len(objects) + len(cpmap)) == max { pastMax = true } diff --git a/backend/walk_test.go b/backend/walk_test.go index 35bdb68..a7b3101 100644 --- a/backend/walk_test.go +++ b/backend/walk_test.go @@ -15,6 +15,9 @@ package backend_test import ( + "crypto/md5" + "encoding/hex" + "fmt" "io/fs" "testing" "testing/fstest" @@ -26,10 +29,44 @@ import ( type walkTest struct { fsys fs.FS expected backend.WalkResults - dc backend.DirObjCheck + getobj backend.GetObjFunc } -func gettag(string) (string, error) { return "myetag", nil } +func getObj(path string, d fs.DirEntry) (types.Object, error) { + if d.IsDir() { + etag := getMD5(path) + + fi, err := d.Info() + if err != nil { + return types.Object{}, fmt.Errorf("get fileinfo: %w", err) + } + + return types.Object{ + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(fi.ModTime()), + }, nil + } + + etag := getMD5(path) + + fi, err := d.Info() + if err != nil { + return types.Object{}, fmt.Errorf("get fileinfo: %w", err) + } + + return types.Object{ + ETag: &etag, + Key: &path, + LastModified: backend.GetTimePtr(fi.ModTime()), + Size: fi.Size(), + }, nil +} + +func getMD5(text string) string { + hash := md5.Sum([]byte(text)) + return hex.EncodeToString(hash[:]) +} func TestWalk(t *testing.T) { tests := []walkTest{ @@ -51,7 +88,7 @@ func TestWalk(t *testing.T) { Key: backend.GetStringPtr("sample.jpg"), }}, }, - dc: func(string) (bool, error) { return false, nil }, + getobj: getObj, }, { // test case single dir/single file @@ -64,12 +101,12 @@ func TestWalk(t *testing.T) { }}, Objects: []types.Object{}, }, - dc: func(string) (bool, error) { return true, nil }, + getobj: getObj, }, } for _, tt := range tests { - res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.dc, gettag, []string{}) + res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{}) if err != nil { t.Fatalf("walk: %v", err) }