diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 92810c6..7914600 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -40,11 +40,12 @@ import ( ) type ScoutfsOpts struct { - ChownUID bool - ChownGID bool - GlacierMode bool - BucketLinks bool - NewDirPerm fs.FileMode + ChownUID bool + ChownGID bool + GlacierMode bool + BucketLinks bool + NewDirPerm fs.FileMode + DisableNoArchive bool } type ScoutFS struct { @@ -78,6 +79,11 @@ type ScoutFS struct { // newDirPerm is the permissions to use when creating new directories newDirPerm fs.FileMode + + // disableNoArchive is used to disable setting scoutam noarchive flag + // on mutlipart parts. This is enabled by default to prevent archive + // copies of temporary multipart parts. + disableNoArchive bool } var _ backend.Backend = &ScoutFS{} @@ -156,6 +162,31 @@ func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) { return uid, gid, needsChown } +func (s *ScoutFS) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { + out, err := s.Posix.UploadPart(ctx, input) + if err != nil { + return nil, err + } + + if !s.disableNoArchive { + sum := sha256.Sum256([]byte(*input.Key)) + partPath := filepath.Join( + *input.Bucket, // bucket + metaTmpMultipartDir, // temp multipart dir + fmt.Sprintf("%x", sum), // hashed objname + *input.UploadId, // upload id + fmt.Sprintf("%v", *input.PartNumber), // part number + ) + + err = setNoArchive(partPath) + if err != nil { + return nil, fmt.Errorf("set noarchive: %w", err) + } + } + + return out, err +} + // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks // ioctl to not have to read and copy the part data to the final object. This // saves a read and write cycle for all mutlipart uploads. @@ -932,30 +963,6 @@ func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) return nil } -func setStaging(objname string) error { - b, err := xattr.Get(objname, flagskey) - if err != nil && !isNoAttr(err) { - return err - } - - var oldflags uint64 - if !isNoAttr(err) { - err = json.Unmarshal(b, &oldflags) - if err != nil { - return err - } - } - - newflags := oldflags | Staging - - if newflags == oldflags { - // no flags change, just return - return nil - } - - return fSetNewGlobalFlags(objname, newflags) -} - func isStaging(objname string) (bool, error) { b, err := xattr.Get(objname, flagskey) if err != nil && !isNoAttr(err) { @@ -973,8 +980,28 @@ func isStaging(objname string) (bool, error) { return flags&Staging == Staging, nil } -func fSetNewGlobalFlags(objname string, flags uint64) error { - b, err := json.Marshal(&flags) +func setFlag(objname string, flag uint64) error { + b, err := xattr.Get(objname, flagskey) + if err != nil && !isNoAttr(err) { + return err + } + + var oldflags uint64 + if !isNoAttr(err) { + err = json.Unmarshal(b, &oldflags) + if err != nil { + return err + } + } + + newflags := oldflags | flag + + if newflags == oldflags { + // no flags change, just return + return nil + } + + b, err = json.Marshal(&newflags) if err != nil { return err } @@ -982,6 +1009,14 @@ func fSetNewGlobalFlags(objname string, flags uint64) error { return xattr.Set(objname, flagskey, b) } +func setStaging(objname string) error { + return setFlag(objname, Staging) +} + +func setNoArchive(objname string) error { + return setFlag(objname, NoArchive) +} + func isNoAttr(err error) bool { xerr, ok := err.(*xattr.Error) if ok && xerr.Err == xattr.ENOATTR { diff --git a/backend/scoutfs/scoutfs_compat.go b/backend/scoutfs/scoutfs_compat.go index 9757710..7aeff65 100644 --- a/backend/scoutfs/scoutfs_compat.go +++ b/backend/scoutfs/scoutfs_compat.go @@ -52,14 +52,15 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) { } return &ScoutFS{ - Posix: p, - rootfd: f, - rootdir: rootdir, - meta: metastore, - chownuid: opts.ChownUID, - chowngid: opts.ChownGID, - glaciermode: opts.GlacierMode, - newDirPerm: opts.NewDirPerm, + Posix: p, + rootfd: f, + rootdir: rootdir, + meta: metastore, + chownuid: opts.ChownUID, + chowngid: opts.ChownGID, + glaciermode: opts.GlacierMode, + newDirPerm: opts.NewDirPerm, + disableNoArchive: opts.DisableNoArchive, }, nil } diff --git a/cmd/versitygw/scoutfs.go b/cmd/versitygw/scoutfs.go index 93274a5..c26b426 100644 --- a/cmd/versitygw/scoutfs.go +++ b/cmd/versitygw/scoutfs.go @@ -24,7 +24,8 @@ import ( ) var ( - glacier bool + glacier bool + disableNoArchive bool ) func scoutfsCommand() *cli.Command { @@ -79,6 +80,12 @@ move interfaces as well as support for tiered filesystems.`, DefaultText: "0755", Value: 0755, }, + &cli.BoolFlag{ + Name: "disable-noarchive", + Usage: "disable setting noarchive for multipart part uploads", + EnvVars: []string{"VGW_DISABLE_NOARCHIVE"}, + Destination: &disableNoArchive, + }, }, } } @@ -98,6 +105,7 @@ func runScoutfs(ctx *cli.Context) error { opts.ChownGID = chowngid opts.BucketLinks = bucketlinks opts.NewDirPerm = fs.FileMode(dirPerms) + opts.DisableNoArchive = disableNoArchive be, err := scoutfs.New(ctx.Args().Get(0), opts) if err != nil {