refactor walk to allow for more general obj translation

backend/posix/posix.go

@@ -1031,19 +1031,7 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (
 
 	fileSystem := os.DirFS(bucket)
 	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
-		func(path string) (bool, error) {
-			_, err := xattr.Get(filepath.Join(bucket, path), etagkey)
-			if isNoAttr(err) {
-				return false, nil
-			}
-			if err != nil {
-				return false, err
-			}
-			return true, nil
-		}, func(path string) (string, error) {
-			etag, err := xattr.Get(filepath.Join(bucket, path), etagkey)
-			return string(etag), err
-		}, []string{metaTmpDir})
+		fileToObj(bucket), []string{metaTmpDir})
 	if err != nil {
 		return nil, fmt.Errorf("walk %v: %w", bucket, err)
 	}
@@ -1061,6 +1049,65 @@ func (p *Posix) ListObjects(bucket, prefix, marker, delim string, maxkeys int) (
 	}, nil
 }
 
+func fileToObj(bucket string) backend.GetObjFunc {
+	return func(path string, d fs.DirEntry) (types.Object, error) {
+		if d.IsDir() {
+			// directory object only happens if directory empty
+			// check to see if this is a directory object by checking etag
+			etagBytes, err := xattr.Get(filepath.Join(bucket, path), etagkey)
+			if isNoAttr(err) || errors.Is(err, fs.ErrNotExist) {
+				return types.Object{}, backend.ErrSkipObj
+			}
+			if err != nil {
+				return types.Object{}, fmt.Errorf("get etag: %w", err)
+			}
+			etag := string(etagBytes)
+
+			fi, err := d.Info()
+			if errors.Is(err, fs.ErrNotExist) {
+				return types.Object{}, backend.ErrSkipObj
+			}
+			if err != nil {
+				return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+			}
+
+			return types.Object{
+				ETag:         &etag,
+				Key:          &path,
+				LastModified: backend.GetTimePtr(fi.ModTime()),
+			}, nil
+		}
+
+		// file object, get object info and fill out object data
+		etagBytes, err := xattr.Get(filepath.Join(bucket, path), etagkey)
+		if errors.Is(err, fs.ErrNotExist) {
+			return types.Object{}, backend.ErrSkipObj
+		}
+		if err != nil && !isNoAttr(err) {
+			return types.Object{}, fmt.Errorf("get etag: %w", err)
+		}
+		// note: isNoAttr(err) will return etagBytes = []byte{}
+		// so this will just set etag to "" if its not already set
+
+		etag := string(etagBytes)
+
+		fi, err := d.Info()
+		if errors.Is(err, fs.ErrNotExist) {
+			return types.Object{}, backend.ErrSkipObj
+		}
+		if err != nil {
+			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		}
+
+		return types.Object{
+			ETag:         &etag,
+			Key:          &path,
+			LastModified: backend.GetTimePtr(fi.ModTime()),
+			Size:         fi.Size(),
+		}, nil
+	}
+}
+
 func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int) (*s3.ListObjectsV2Output, error) {
 	_, err := os.Stat(bucket)
 	if errors.Is(err, fs.ErrNotExist) {
@@ -1072,19 +1119,7 @@ func (p *Posix) ListObjectsV2(bucket, prefix, marker, delim string, maxkeys int)
 
 	fileSystem := os.DirFS(bucket)
 	results, err := backend.Walk(fileSystem, prefix, delim, marker, maxkeys,
-		func(path string) (bool, error) {
-			_, err := xattr.Get(filepath.Join(bucket, path), etagkey)
-			if isNoAttr(err) {
-				return false, nil
-			}
-			if err != nil {
-				return false, err
-			}
-			return true, nil
-		}, func(path string) (string, error) {
-			etag, err := xattr.Get(filepath.Join(bucket, path), etagkey)
-			return string(etag), err
-		}, []string{metaTmpDir})
+		fileToObj(bucket), []string{metaTmpDir})
 	if err != nil {
 		return nil, fmt.Errorf("walk %v: %w", bucket, err)
 	}

backend/walk.go

@@ -15,6 +15,7 @@
 package backend
 
 import (
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -31,12 +32,13 @@ type WalkResults struct {
 	NextMarker     string
 }
 
-type DirObjCheck func(path string) (bool, error)
-type GetETag func(path string) (string, error)
+type GetObjFunc func(path string, d fs.DirEntry) (types.Object, error)
+
+var ErrSkipObj = errors.New("skip this object")
 
 // Walk walks the supplied fs.FS and returns results compatible with list
 // objects responses
-func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, dirchk DirObjCheck, getetag GetETag, skipdirs []string) (WalkResults, error) {
+func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, getObj GetObjFunc, skipdirs []string) (WalkResults, error) {
 	cpmap := make(map[string]struct{})
 	var objects []types.Object
 
@@ -90,26 +92,14 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, dirchk Di
 				return fmt.Errorf("readdir %q: %w", path, err)
 			}
 			if len(ents) == 0 {
-				dirobj, err := dirchk(path)
+				dirobj, err := getObj(path, d)
+				if err == ErrSkipObj {
+					return nil
+				}
 				if err != nil {
-					return fmt.Errorf("directory object check %q: %w", path, err)
-				}
-				if dirobj {
-					fi, err := d.Info()
-					if err != nil {
-						return fmt.Errorf("dir info %q: %w", path, err)
-					}
-					etag, err := getetag(path)
-					if err != nil {
-						return fmt.Errorf("get etag %q: %w", path, err)
-					}
-					path := path + "/"
-					objects = append(objects, types.Object{
-						ETag:         &etag,
-						Key:          &path,
-						LastModified: GetTimePtr(fi.ModTime()),
-					})
+					return fmt.Errorf("directory to object %q: %w", path, err)
 				}
+				objects = append(objects, dirobj)
 			}
 
 			return nil
@@ -130,21 +120,14 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, dirchk Di
 		if delimiter == "" {
 			// If no delimeter specified, then all files with matching
 			// prefix are included in results
-			fi, err := d.Info()
-			if err != nil {
-				return fmt.Errorf("get info for %v: %w", path, err)
+			obj, err := getObj(path, d)
+			if err == ErrSkipObj {
+				return nil
 			}
-			etag, err := getetag(path)
 			if err != nil {
-				return fmt.Errorf("get etag %q: %w", path, err)
+				return fmt.Errorf("file to object %q: %w", path, err)
 			}
-
-			objects = append(objects, types.Object{
-				ETag:         &etag,
-				Key:          &path,
-				LastModified: GetTimePtr(fi.ModTime()),
-				Size:         fi.Size(),
-			})
+			objects = append(objects, obj)
 
 			if max > 0 && (len(objects)+len(cpmap)) == max {
 				pastMax = true
@@ -177,20 +160,14 @@ func Walk(fileSystem fs.FS, prefix, delimiter, marker string, max int, dirchk Di
 		suffix := strings.TrimPrefix(path, prefix)
 		before, _, found := strings.Cut(suffix, delimiter)
 		if !found {
-			fi, err := d.Info()
-			if err != nil {
-				return fmt.Errorf("get info for %v: %w", path, err)
+			obj, err := getObj(path, d)
+			if err == ErrSkipObj {
+				return nil
 			}
-			etag, err := getetag(path)
 			if err != nil {
-				return fmt.Errorf("get etag %q: %w", path, err)
+				return fmt.Errorf("file to object %q: %w", path, err)
 			}
-			objects = append(objects, types.Object{
-				ETag:         &etag,
-				Key:          &path,
-				LastModified: GetTimePtr(fi.ModTime()),
-				Size:         fi.Size(),
-			})
+			objects = append(objects, obj)
 			if (len(objects) + len(cpmap)) == max {
 				pastMax = true
 			}

backend/walk_test.go

@@ -15,6 +15,9 @@
 package backend_test
 
 import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
 	"io/fs"
 	"testing"
 	"testing/fstest"
@@ -26,10 +29,44 @@ import (
 type walkTest struct {
 	fsys     fs.FS
 	expected backend.WalkResults
-	dc       backend.DirObjCheck
+	getobj   backend.GetObjFunc
 }
 
-func gettag(string) (string, error) { return "myetag", nil }
+func getObj(path string, d fs.DirEntry) (types.Object, error) {
+	if d.IsDir() {
+		etag := getMD5(path)
+
+		fi, err := d.Info()
+		if err != nil {
+			return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+		}
+
+		return types.Object{
+			ETag:         &etag,
+			Key:          &path,
+			LastModified: backend.GetTimePtr(fi.ModTime()),
+		}, nil
+	}
+
+	etag := getMD5(path)
+
+	fi, err := d.Info()
+	if err != nil {
+		return types.Object{}, fmt.Errorf("get fileinfo: %w", err)
+	}
+
+	return types.Object{
+		ETag:         &etag,
+		Key:          &path,
+		LastModified: backend.GetTimePtr(fi.ModTime()),
+		Size:         fi.Size(),
+	}, nil
+}
+
+func getMD5(text string) string {
+	hash := md5.Sum([]byte(text))
+	return hex.EncodeToString(hash[:])
+}
 
 func TestWalk(t *testing.T) {
 	tests := []walkTest{
@@ -51,7 +88,7 @@ func TestWalk(t *testing.T) {
 					Key: backend.GetStringPtr("sample.jpg"),
 				}},
 			},
-			dc: func(string) (bool, error) { return false, nil },
+			getobj: getObj,
 		},
 		{
 			// test case single dir/single file
@@ -64,12 +101,12 @@ func TestWalk(t *testing.T) {
 				}},
 				Objects: []types.Object{},
 			},
-			dc: func(string) (bool, error) { return true, nil },
+			getobj: getObj,
 		},
 	}
 
 	for _, tt := range tests {
-		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.dc, gettag, []string{})
+		res, err := backend.Walk(tt.fsys, "", "/", "", 1000, tt.getobj, []string{})
 		if err != nil {
 			t.Fatalf("walk: %v", err)
 		}