From d034f87f6029790ef398a171b90a52d4e95b6c9f Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Sat, 8 Mar 2025 10:22:43 -0800 Subject: [PATCH] feat: add noarchive to scoutfs part files The part files for multipart uploads are considered temporary files and should not be archived by default. This adds the noarchive attribute to the part files to prevent scoutam from trying to archive these. There is a new parameter, disablenoarchive, that will prevent adding the noarchive attribute to these files for the case where there is a desire to archive these temp files. --- backend/scoutfs/scoutfs.go | 97 +++++++++++++++++++++---------- backend/scoutfs/scoutfs_compat.go | 17 +++--- cmd/versitygw/scoutfs.go | 10 +++- 3 files changed, 84 insertions(+), 40 deletions(-) diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 92810c6..7914600 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -40,11 +40,12 @@ import ( ) type ScoutfsOpts struct { - ChownUID bool - ChownGID bool - GlacierMode bool - BucketLinks bool - NewDirPerm fs.FileMode + ChownUID bool + ChownGID bool + GlacierMode bool + BucketLinks bool + NewDirPerm fs.FileMode + DisableNoArchive bool } type ScoutFS struct { @@ -78,6 +79,11 @@ type ScoutFS struct { // newDirPerm is the permissions to use when creating new directories newDirPerm fs.FileMode + + // disableNoArchive is used to disable setting scoutam noarchive flag + // on mutlipart parts. This is enabled by default to prevent archive + // copies of temporary multipart parts. + disableNoArchive bool } var _ backend.Backend = &ScoutFS{} @@ -156,6 +162,31 @@ func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) { return uid, gid, needsChown } +func (s *ScoutFS) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { + out, err := s.Posix.UploadPart(ctx, input) + if err != nil { + return nil, err + } + + if !s.disableNoArchive { + sum := sha256.Sum256([]byte(*input.Key)) + partPath := filepath.Join( + *input.Bucket, // bucket + metaTmpMultipartDir, // temp multipart dir + fmt.Sprintf("%x", sum), // hashed objname + *input.UploadId, // upload id + fmt.Sprintf("%v", *input.PartNumber), // part number + ) + + err = setNoArchive(partPath) + if err != nil { + return nil, fmt.Errorf("set noarchive: %w", err) + } + } + + return out, err +} + // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks // ioctl to not have to read and copy the part data to the final object. This // saves a read and write cycle for all mutlipart uploads. @@ -932,30 +963,6 @@ func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) return nil } -func setStaging(objname string) error { - b, err := xattr.Get(objname, flagskey) - if err != nil && !isNoAttr(err) { - return err - } - - var oldflags uint64 - if !isNoAttr(err) { - err = json.Unmarshal(b, &oldflags) - if err != nil { - return err - } - } - - newflags := oldflags | Staging - - if newflags == oldflags { - // no flags change, just return - return nil - } - - return fSetNewGlobalFlags(objname, newflags) -} - func isStaging(objname string) (bool, error) { b, err := xattr.Get(objname, flagskey) if err != nil && !isNoAttr(err) { @@ -973,8 +980,28 @@ func isStaging(objname string) (bool, error) { return flags&Staging == Staging, nil } -func fSetNewGlobalFlags(objname string, flags uint64) error { - b, err := json.Marshal(&flags) +func setFlag(objname string, flag uint64) error { + b, err := xattr.Get(objname, flagskey) + if err != nil && !isNoAttr(err) { + return err + } + + var oldflags uint64 + if !isNoAttr(err) { + err = json.Unmarshal(b, &oldflags) + if err != nil { + return err + } + } + + newflags := oldflags | flag + + if newflags == oldflags { + // no flags change, just return + return nil + } + + b, err = json.Marshal(&newflags) if err != nil { return err } @@ -982,6 +1009,14 @@ func fSetNewGlobalFlags(objname string, flags uint64) error { return xattr.Set(objname, flagskey, b) } +func setStaging(objname string) error { + return setFlag(objname, Staging) +} + +func setNoArchive(objname string) error { + return setFlag(objname, NoArchive) +} + func isNoAttr(err error) bool { xerr, ok := err.(*xattr.Error) if ok && xerr.Err == xattr.ENOATTR { diff --git a/backend/scoutfs/scoutfs_compat.go b/backend/scoutfs/scoutfs_compat.go index 9757710..7aeff65 100644 --- a/backend/scoutfs/scoutfs_compat.go +++ b/backend/scoutfs/scoutfs_compat.go @@ -52,14 +52,15 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) { } return &ScoutFS{ - Posix: p, - rootfd: f, - rootdir: rootdir, - meta: metastore, - chownuid: opts.ChownUID, - chowngid: opts.ChownGID, - glaciermode: opts.GlacierMode, - newDirPerm: opts.NewDirPerm, + Posix: p, + rootfd: f, + rootdir: rootdir, + meta: metastore, + chownuid: opts.ChownUID, + chowngid: opts.ChownGID, + glaciermode: opts.GlacierMode, + newDirPerm: opts.NewDirPerm, + disableNoArchive: opts.DisableNoArchive, }, nil } diff --git a/cmd/versitygw/scoutfs.go b/cmd/versitygw/scoutfs.go index 93274a5..c26b426 100644 --- a/cmd/versitygw/scoutfs.go +++ b/cmd/versitygw/scoutfs.go @@ -24,7 +24,8 @@ import ( ) var ( - glacier bool + glacier bool + disableNoArchive bool ) func scoutfsCommand() *cli.Command { @@ -79,6 +80,12 @@ move interfaces as well as support for tiered filesystems.`, DefaultText: "0755", Value: 0755, }, + &cli.BoolFlag{ + Name: "disable-noarchive", + Usage: "disable setting noarchive for multipart part uploads", + EnvVars: []string{"VGW_DISABLE_NOARCHIVE"}, + Destination: &disableNoArchive, + }, }, } } @@ -98,6 +105,7 @@ func runScoutfs(ctx *cli.Context) error { opts.ChownGID = chowngid opts.BucketLinks = bucketlinks opts.NewDirPerm = fs.FileMode(dirPerms) + opts.DisableNoArchive = disableNoArchive be, err := scoutfs.New(ctx.Args().Get(0), opts) if err != nil {