Merge pull request #522 from versity/ben/sidecar_meta

feat: add optional sidecar files for metadata

backend/meta/sidecar.go

@@ -0,0 +1,125 @@
+package meta
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+// SideCar is a metadata storer that uses sidecar files to store metadata.
+type SideCar struct {
+	dir string
+}
+
+const (
+	sidecarmeta = "meta"
+)
+
+// NewSideCar creates a new SideCar metadata storer.
+func NewSideCar(dir string) (SideCar, error) {
+	fi, err := os.Lstat(dir)
+	if err != nil {
+		return SideCar{}, fmt.Errorf("failed to stat directory: %v", err)
+	}
+	if !fi.IsDir() {
+		return SideCar{}, fmt.Errorf("not a directory")
+	}
+
+	return SideCar{dir: dir}, nil
+}
+
+// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket.
+func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	value, err := os.ReadFile(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil, ErrNoSuchKey
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to read attribute: %v", err)
+	}
+
+	return value, nil
+}
+
+// StoreAttribute stores the value of a specific attribute for an object or a bucket.
+func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	err := os.MkdirAll(metadir, 0777)
+	if err != nil {
+		return fmt.Errorf("failed to create metadata directory: %v", err)
+	}
+
+	attr := filepath.Join(metadir, attribute)
+	err = os.WriteFile(attr, value, 0666)
+	if err != nil {
+		return fmt.Errorf("failed to write attribute: %v", err)
+	}
+
+	return nil
+}
+
+// DeleteAttribute removes the value of a specific attribute for an object or a bucket.
+func (s SideCar) DeleteAttribute(bucket, object, attribute string) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+	attr := filepath.Join(metadir, attribute)
+
+	err := os.Remove(attr)
+	if errors.Is(err, os.ErrNotExist) {
+		return ErrNoSuchKey
+	}
+	if err != nil {
+		return fmt.Errorf("failed to remove attribute: %v", err)
+	}
+
+	return nil
+}
+
+// ListAttributes lists all attributes for an object or a bucket.
+func (s SideCar) ListAttributes(bucket, object string) ([]string, error) {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+
+	ents, err := os.ReadDir(metadir)
+	if errors.Is(err, os.ErrNotExist) {
+		return []string{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to list attributes: %v", err)
+	}
+
+	var attrs []string
+	for _, ent := range ents {
+		attrs = append(attrs, ent.Name())
+	}
+
+	return attrs, nil
+}
+
+// DeleteAttributes removes all attributes for an object or a bucket.
+func (s SideCar) DeleteAttributes(bucket, object string) error {
+	metadir := filepath.Join(s.dir, bucket, object, sidecarmeta)
+	if object == "" {
+		metadir = filepath.Join(s.dir, bucket, sidecarmeta)
+	}
+
+	err := os.RemoveAll(metadir)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("failed to remove attributes: %v", err)
+	}
+	return nil
+}

backend/posix/posix.go

@@ -107,9 +107,14 @@ type PosixOpts struct {
 	BucketLinks   bool
 	VersioningDir string
 	NewDirPerm    fs.FileMode
+	SideCarDir    string
 }
 
 func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) {
+	if opts.SideCarDir != "" && strings.HasPrefix(opts.SideCarDir, rootdir) {
+		return nil, fmt.Errorf("sidecar directory cannot be inside the gateway root directory")
+	}
+
 	err := os.Chdir(rootdir)
 	if err != nil {
 		return nil, fmt.Errorf("chdir %v: %w", rootdir, err)
@@ -120,46 +125,36 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
 		return nil, fmt.Errorf("open %v: %w", rootdir, err)
 	}
 
-	var verioningdirAbs string
+	rootdirAbs, err := filepath.Abs(rootdir)
+	if err != nil {
+		return nil, fmt.Errorf("get absolute path of %v: %w", rootdir, err)
+	}
 
+	var verioningdirAbs string
 	// Ensure the versioning directory isn't within the root directory
 	if opts.VersioningDir != "" {
-		rootdirAbs, err := filepath.Abs(rootdir)
+		verioningdirAbs, err = validateSubDir(rootdirAbs, opts.VersioningDir)
 		if err != nil {
-			return nil, fmt.Errorf("get absolute path of %v: %w", rootdir, err)
-		}
-
-		verioningdirAbs, err = filepath.Abs(opts.VersioningDir)
-		if err != nil {
-			return nil, fmt.Errorf("get absolute path of %v: %w", opts.VersioningDir, err)
-		}
-
-		// Ensure the paths end with a separator
-		if !strings.HasSuffix(rootdirAbs, string(filepath.Separator)) {
-			rootdirAbs += string(filepath.Separator)
-		}
-
-		if !strings.HasSuffix(verioningdirAbs, string(filepath.Separator)) {
-			verioningdirAbs += string(filepath.Separator)
-		}
-
-		// Ensure the posix root directory doesn't contain the versioning directory
-		if strings.HasPrefix(verioningdirAbs, rootdirAbs) {
-			return nil, fmt.Errorf("the root directory %v contains the versioning directory %v", rootdir, opts.VersioningDir)
-		}
-
-		vDir, err := os.Stat(verioningdirAbs)
-		if err != nil {
-			return nil, fmt.Errorf("stat versioning dir: %w", err)
-		}
-
-		// Check the versioning path to be a directory
-		if !vDir.IsDir() {
-			return nil, fmt.Errorf("versioning path should be a directory")
+			return nil, err
 		}
 	}
 
-	fmt.Printf("Bucket versioning enabled with directory: %v\n", verioningdirAbs)
+	var sidecardirAbs string
+	// Ensure the sidecar directory isn't within the root directory
+	if opts.SideCarDir != "" {
+		sidecardirAbs, err = validateSubDir(rootdirAbs, opts.SideCarDir)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if verioningdirAbs != "" {
+		fmt.Println("Bucket versioning enabled with directory:", verioningdirAbs)
+	}
+
+	if sidecardirAbs != "" {
+		fmt.Println("Using sidecar directory for metadata:", sidecardirAbs)
+	}
 
 	return &Posix{
 		meta:          meta,
@@ -175,6 +170,48 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
 	}, nil
 }
 
+func validateSubDir(root, dir string) (string, error) {
+	absDir, err := filepath.Abs(dir)
+	if err != nil {
+		return "", fmt.Errorf("get absolute path of %v: %w",
+			dir, err)
+	}
+
+	if isDirBelowRoot(root, absDir) {
+		return "", fmt.Errorf("the root directory %v contains the directory %v",
+			root, dir)
+	}
+
+	vDir, err := os.Stat(absDir)
+	if err != nil {
+		return "", fmt.Errorf("stat %q: %w", absDir, err)
+	}
+
+	if !vDir.IsDir() {
+		return "", fmt.Errorf("path %q is not a directory", absDir)
+	}
+
+	return absDir, nil
+}
+
+func isDirBelowRoot(root, dir string) bool {
+	// Ensure the paths ends with a separator
+	if !strings.HasSuffix(root, string(filepath.Separator)) {
+		root += string(filepath.Separator)
+	}
+
+	if !strings.HasSuffix(dir, string(filepath.Separator)) {
+		dir += string(filepath.Separator)
+	}
+
+	// Ensure the root directory doesn't contain the directory
+	if strings.HasPrefix(dir, root) {
+		return true
+	}
+
+	return false
+}
+
 func (p *Posix) Shutdown() {
 	p.rootfd.Close()
 }

cmd/versitygw/posix.go

@@ -29,6 +29,7 @@ var (
 	bucketlinks        bool
 	versioningDir      string
 	dirPerms           uint
+	sidecar            string
 )
 
 func posixCommand() *cli.Command {
@@ -79,6 +80,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				DefaultText: "0755",
 				Value:       0755,
 			},
+			&cli.StringFlag{
+				Name:        "sidecar",
+				Usage:       "use provided sidecar directory to store metadata",
+				EnvVars:     []string{"VGW_META_SIDECAR"},
+				Destination: &sidecar,
+			},
 		},
 	}
 }
@@ -89,24 +96,39 @@ func runPosix(ctx *cli.Context) error {
 	}
 
 	gwroot := (ctx.Args().Get(0))
-	err := meta.XattrMeta{}.Test(gwroot)
-	if err != nil {
-		return fmt.Errorf("posix xattr check: %v", err)
-	}
 
 	if dirPerms > math.MaxUint32 {
 		return fmt.Errorf("invalid directory permissions: %d", dirPerms)
 	}
 
-	be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{
+	opts := posix.PosixOpts{
 		ChownUID:      chownuid,
 		ChownGID:      chowngid,
 		BucketLinks:   bucketlinks,
 		VersioningDir: versioningDir,
 		NewDirPerm:    fs.FileMode(dirPerms),
-	})
+	}
+
+	var ms meta.MetadataStorer
+	switch {
+	case sidecar != "":
+		sc, err := meta.NewSideCar(sidecar)
+		if err != nil {
+			return fmt.Errorf("failed to init sidecar metadata: %w", err)
+		}
+		ms = sc
+		opts.SideCarDir = sidecar
+	default:
+		ms = meta.XattrMeta{}
+		err := meta.XattrMeta{}.Test(gwroot)
+		if err != nil {
+			return fmt.Errorf("xattr check failed: %w", err)
+		}
+	}
+
+	be, err := posix.New(gwroot, ms, opts)
 	if err != nil {
-		return fmt.Errorf("init posix: %v", err)
+		return fmt.Errorf("failed to init posix backend: %w", err)
 	}
 
 	return runGateway(ctx.Context, be)