From 4025172897a74dcd611c1b38086c2fe99d224430 Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Tue, 16 Apr 2024 16:52:22 -0700 Subject: [PATCH] feat: add optional sidecar files for metadata This adds the option to store metadata for objects and buckets within a specified directory: bucket: //meta/ object: /bucket//meta/ Example invocation: ./versitygw -a myaccess -s mysecret posix --sidecar /tmp/sidecar /tmp/gw The attributes are stored by name within the hidden directory. --- backend/meta/sidecar.go | 125 ++++++++++++++++++++++++++++++++++++++++ backend/posix/posix.go | 103 ++++++++++++++++++++++----------- cmd/versitygw/posix.go | 36 +++++++++--- 3 files changed, 224 insertions(+), 40 deletions(-) create mode 100644 backend/meta/sidecar.go diff --git a/backend/meta/sidecar.go b/backend/meta/sidecar.go new file mode 100644 index 0000000..cf19665 --- /dev/null +++ b/backend/meta/sidecar.go @@ -0,0 +1,125 @@ +package meta + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// SideCar is a metadata storer that uses sidecar files to store metadata. +type SideCar struct { + dir string +} + +const ( + sidecarmeta = "meta" +) + +// NewSideCar creates a new SideCar metadata storer. +func NewSideCar(dir string) (SideCar, error) { + fi, err := os.Lstat(dir) + if err != nil { + return SideCar{}, fmt.Errorf("failed to stat directory: %v", err) + } + if !fi.IsDir() { + return SideCar{}, fmt.Errorf("not a directory") + } + + return SideCar{dir: dir}, nil +} + +// RetrieveAttribute retrieves the value of a specific attribute for an object or a bucket. +func (s SideCar) RetrieveAttribute(_ *os.File, bucket, object, attribute string) ([]byte, error) { + metadir := filepath.Join(s.dir, bucket, object, sidecarmeta) + if object == "" { + metadir = filepath.Join(s.dir, bucket, sidecarmeta) + } + attr := filepath.Join(metadir, attribute) + + value, err := os.ReadFile(attr) + if errors.Is(err, os.ErrNotExist) { + return nil, ErrNoSuchKey + } + if err != nil { + return nil, fmt.Errorf("failed to read attribute: %v", err) + } + + return value, nil +} + +// StoreAttribute stores the value of a specific attribute for an object or a bucket. +func (s SideCar) StoreAttribute(_ *os.File, bucket, object, attribute string, value []byte) error { + metadir := filepath.Join(s.dir, bucket, object, sidecarmeta) + if object == "" { + metadir = filepath.Join(s.dir, bucket, sidecarmeta) + } + err := os.MkdirAll(metadir, 0777) + if err != nil { + return fmt.Errorf("failed to create metadata directory: %v", err) + } + + attr := filepath.Join(metadir, attribute) + err = os.WriteFile(attr, value, 0666) + if err != nil { + return fmt.Errorf("failed to write attribute: %v", err) + } + + return nil +} + +// DeleteAttribute removes the value of a specific attribute for an object or a bucket. +func (s SideCar) DeleteAttribute(bucket, object, attribute string) error { + metadir := filepath.Join(s.dir, bucket, object, sidecarmeta) + if object == "" { + metadir = filepath.Join(s.dir, bucket, sidecarmeta) + } + attr := filepath.Join(metadir, attribute) + + err := os.Remove(attr) + if errors.Is(err, os.ErrNotExist) { + return ErrNoSuchKey + } + if err != nil { + return fmt.Errorf("failed to remove attribute: %v", err) + } + + return nil +} + +// ListAttributes lists all attributes for an object or a bucket. +func (s SideCar) ListAttributes(bucket, object string) ([]string, error) { + metadir := filepath.Join(s.dir, bucket, object, sidecarmeta) + if object == "" { + metadir = filepath.Join(s.dir, bucket, sidecarmeta) + } + + ents, err := os.ReadDir(metadir) + if errors.Is(err, os.ErrNotExist) { + return []string{}, nil + } + if err != nil { + return nil, fmt.Errorf("failed to list attributes: %v", err) + } + + var attrs []string + for _, ent := range ents { + attrs = append(attrs, ent.Name()) + } + + return attrs, nil +} + +// DeleteAttributes removes all attributes for an object or a bucket. +func (s SideCar) DeleteAttributes(bucket, object string) error { + metadir := filepath.Join(s.dir, bucket, object, sidecarmeta) + if object == "" { + metadir = filepath.Join(s.dir, bucket, sidecarmeta) + } + + err := os.RemoveAll(metadir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to remove attributes: %v", err) + } + return nil +} diff --git a/backend/posix/posix.go b/backend/posix/posix.go index b2e8609..53ad238 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -107,9 +107,14 @@ type PosixOpts struct { BucketLinks bool VersioningDir string NewDirPerm fs.FileMode + SideCarDir string } func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) { + if opts.SideCarDir != "" && strings.HasPrefix(opts.SideCarDir, rootdir) { + return nil, fmt.Errorf("sidecar directory cannot be inside the gateway root directory") + } + err := os.Chdir(rootdir) if err != nil { return nil, fmt.Errorf("chdir %v: %w", rootdir, err) @@ -120,46 +125,36 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro return nil, fmt.Errorf("open %v: %w", rootdir, err) } - var verioningdirAbs string + rootdirAbs, err := filepath.Abs(rootdir) + if err != nil { + return nil, fmt.Errorf("get absolute path of %v: %w", rootdir, err) + } + var verioningdirAbs string // Ensure the versioning directory isn't within the root directory if opts.VersioningDir != "" { - rootdirAbs, err := filepath.Abs(rootdir) + verioningdirAbs, err = validateSubDir(rootdirAbs, opts.VersioningDir) if err != nil { - return nil, fmt.Errorf("get absolute path of %v: %w", rootdir, err) - } - - verioningdirAbs, err = filepath.Abs(opts.VersioningDir) - if err != nil { - return nil, fmt.Errorf("get absolute path of %v: %w", opts.VersioningDir, err) - } - - // Ensure the paths end with a separator - if !strings.HasSuffix(rootdirAbs, string(filepath.Separator)) { - rootdirAbs += string(filepath.Separator) - } - - if !strings.HasSuffix(verioningdirAbs, string(filepath.Separator)) { - verioningdirAbs += string(filepath.Separator) - } - - // Ensure the posix root directory doesn't contain the versioning directory - if strings.HasPrefix(verioningdirAbs, rootdirAbs) { - return nil, fmt.Errorf("the root directory %v contains the versioning directory %v", rootdir, opts.VersioningDir) - } - - vDir, err := os.Stat(verioningdirAbs) - if err != nil { - return nil, fmt.Errorf("stat versioning dir: %w", err) - } - - // Check the versioning path to be a directory - if !vDir.IsDir() { - return nil, fmt.Errorf("versioning path should be a directory") + return nil, err } } - fmt.Printf("Bucket versioning enabled with directory: %v\n", verioningdirAbs) + var sidecardirAbs string + // Ensure the sidecar directory isn't within the root directory + if opts.SideCarDir != "" { + sidecardirAbs, err = validateSubDir(rootdirAbs, opts.SideCarDir) + if err != nil { + return nil, err + } + } + + if verioningdirAbs != "" { + fmt.Println("Bucket versioning enabled with directory:", verioningdirAbs) + } + + if sidecardirAbs != "" { + fmt.Println("Using sidecar directory for metadata:", sidecardirAbs) + } return &Posix{ meta: meta, @@ -175,6 +170,48 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro }, nil } +func validateSubDir(root, dir string) (string, error) { + absDir, err := filepath.Abs(dir) + if err != nil { + return "", fmt.Errorf("get absolute path of %v: %w", + dir, err) + } + + if isDirBelowRoot(root, absDir) { + return "", fmt.Errorf("the root directory %v contains the directory %v", + root, dir) + } + + vDir, err := os.Stat(absDir) + if err != nil { + return "", fmt.Errorf("stat %q: %w", absDir, err) + } + + if !vDir.IsDir() { + return "", fmt.Errorf("path %q is not a directory", absDir) + } + + return absDir, nil +} + +func isDirBelowRoot(root, dir string) bool { + // Ensure the paths ends with a separator + if !strings.HasSuffix(root, string(filepath.Separator)) { + root += string(filepath.Separator) + } + + if !strings.HasSuffix(dir, string(filepath.Separator)) { + dir += string(filepath.Separator) + } + + // Ensure the root directory doesn't contain the directory + if strings.HasPrefix(dir, root) { + return true + } + + return false +} + func (p *Posix) Shutdown() { p.rootfd.Close() } diff --git a/cmd/versitygw/posix.go b/cmd/versitygw/posix.go index db86c19..9ff8177 100644 --- a/cmd/versitygw/posix.go +++ b/cmd/versitygw/posix.go @@ -29,6 +29,7 @@ var ( bucketlinks bool versioningDir string dirPerms uint + sidecar string ) func posixCommand() *cli.Command { @@ -79,6 +80,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`, DefaultText: "0755", Value: 0755, }, + &cli.StringFlag{ + Name: "sidecar", + Usage: "use provided sidecar directory to store metadata", + EnvVars: []string{"VGW_META_SIDECAR"}, + Destination: &sidecar, + }, }, } } @@ -89,24 +96,39 @@ func runPosix(ctx *cli.Context) error { } gwroot := (ctx.Args().Get(0)) - err := meta.XattrMeta{}.Test(gwroot) - if err != nil { - return fmt.Errorf("posix xattr check: %v", err) - } if dirPerms > math.MaxUint32 { return fmt.Errorf("invalid directory permissions: %d", dirPerms) } - be, err := posix.New(gwroot, meta.XattrMeta{}, posix.PosixOpts{ + opts := posix.PosixOpts{ ChownUID: chownuid, ChownGID: chowngid, BucketLinks: bucketlinks, VersioningDir: versioningDir, NewDirPerm: fs.FileMode(dirPerms), - }) + } + + var ms meta.MetadataStorer + switch { + case sidecar != "": + sc, err := meta.NewSideCar(sidecar) + if err != nil { + return fmt.Errorf("failed to init sidecar metadata: %w", err) + } + ms = sc + opts.SideCarDir = sidecar + default: + ms = meta.XattrMeta{} + err := meta.XattrMeta{}.Test(gwroot) + if err != nil { + return fmt.Errorf("xattr check failed: %w", err) + } + } + + be, err := posix.New(gwroot, ms, opts) if err != nil { - return fmt.Errorf("init posix: %v", err) + return fmt.Errorf("failed to init posix backend: %w", err) } return runGateway(ctx.Context, be)