Merge pull request #1129 from versity/ben/scoutfs-noarchive

feat: add noarchive to scoutfs part files

backend/scoutfs/scoutfs.go

@@ -40,11 +40,12 @@ import (
 )
 
 type ScoutfsOpts struct {
-	ChownUID    bool
-	ChownGID    bool
-	GlacierMode bool
-	BucketLinks bool
-	NewDirPerm  fs.FileMode
+	ChownUID         bool
+	ChownGID         bool
+	GlacierMode      bool
+	BucketLinks      bool
+	NewDirPerm       fs.FileMode
+	DisableNoArchive bool
 }
 
 type ScoutFS struct {
@@ -78,6 +79,11 @@ type ScoutFS struct {
 
 	// newDirPerm is the permissions to use when creating new directories
 	newDirPerm fs.FileMode
+
+	// disableNoArchive is used to disable setting scoutam noarchive flag
+	// on mutlipart parts. This is enabled by default to prevent archive
+	// copies of temporary multipart parts.
+	disableNoArchive bool
 }
 
 var _ backend.Backend = &ScoutFS{}
@@ -156,6 +162,31 @@ func (s *ScoutFS) getChownIDs(acct auth.Account) (int, int, bool) {
 	return uid, gid, needsChown
 }
 
+func (s *ScoutFS) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) {
+	out, err := s.Posix.UploadPart(ctx, input)
+	if err != nil {
+		return nil, err
+	}
+
+	if !s.disableNoArchive {
+		sum := sha256.Sum256([]byte(*input.Key))
+		partPath := filepath.Join(
+			*input.Bucket,                        // bucket
+			metaTmpMultipartDir,                  // temp multipart dir
+			fmt.Sprintf("%x", sum),               // hashed objname
+			*input.UploadId,                      // upload id
+			fmt.Sprintf("%v", *input.PartNumber), // part number
+		)
+
+		err = setNoArchive(partPath)
+		if err != nil {
+			return nil, fmt.Errorf("set noarchive: %w", err)
+		}
+	}
+
+	return out, err
+}
+
 // CompleteMultipartUpload scoutfs complete upload uses scoutfs move blocks
 // ioctl to not have to read and copy the part data to the final object. This
 // saves a read and write cycle for all mutlipart uploads.
@@ -932,30 +963,6 @@ func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput)
 	return nil
 }
 
-func setStaging(objname string) error {
-	b, err := xattr.Get(objname, flagskey)
-	if err != nil && !isNoAttr(err) {
-		return err
-	}
-
-	var oldflags uint64
-	if !isNoAttr(err) {
-		err = json.Unmarshal(b, &oldflags)
-		if err != nil {
-			return err
-		}
-	}
-
-	newflags := oldflags | Staging
-
-	if newflags == oldflags {
-		// no flags change, just return
-		return nil
-	}
-
-	return fSetNewGlobalFlags(objname, newflags)
-}
-
 func isStaging(objname string) (bool, error) {
 	b, err := xattr.Get(objname, flagskey)
 	if err != nil && !isNoAttr(err) {
@@ -973,8 +980,28 @@ func isStaging(objname string) (bool, error) {
 	return flags&Staging == Staging, nil
 }
 
-func fSetNewGlobalFlags(objname string, flags uint64) error {
-	b, err := json.Marshal(&flags)
+func setFlag(objname string, flag uint64) error {
+	b, err := xattr.Get(objname, flagskey)
+	if err != nil && !isNoAttr(err) {
+		return err
+	}
+
+	var oldflags uint64
+	if !isNoAttr(err) {
+		err = json.Unmarshal(b, &oldflags)
+		if err != nil {
+			return err
+		}
+	}
+
+	newflags := oldflags | flag
+
+	if newflags == oldflags {
+		// no flags change, just return
+		return nil
+	}
+
+	b, err = json.Marshal(&newflags)
 	if err != nil {
 		return err
 	}
@@ -982,6 +1009,14 @@ func fSetNewGlobalFlags(objname string, flags uint64) error {
 	return xattr.Set(objname, flagskey, b)
 }
 
+func setStaging(objname string) error {
+	return setFlag(objname, Staging)
+}
+
+func setNoArchive(objname string) error {
+	return setFlag(objname, NoArchive)
+}
+
 func isNoAttr(err error) bool {
 	xerr, ok := err.(*xattr.Error)
 	if ok && xerr.Err == xattr.ENOATTR {

backend/scoutfs/scoutfs_compat.go

@@ -52,14 +52,15 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 	}
 
 	return &ScoutFS{
-		Posix:       p,
-		rootfd:      f,
-		rootdir:     rootdir,
-		meta:        metastore,
-		chownuid:    opts.ChownUID,
-		chowngid:    opts.ChownGID,
-		glaciermode: opts.GlacierMode,
-		newDirPerm:  opts.NewDirPerm,
+		Posix:            p,
+		rootfd:           f,
+		rootdir:          rootdir,
+		meta:             metastore,
+		chownuid:         opts.ChownUID,
+		chowngid:         opts.ChownGID,
+		glaciermode:      opts.GlacierMode,
+		newDirPerm:       opts.NewDirPerm,
+		disableNoArchive: opts.DisableNoArchive,
 	}, nil
 }
 

cmd/versitygw/scoutfs.go

@@ -24,7 +24,8 @@ import (
 )
 
 var (
-	glacier bool
+	glacier          bool
+	disableNoArchive bool
 )
 
 func scoutfsCommand() *cli.Command {
@@ -79,6 +80,12 @@ move interfaces as well as support for tiered filesystems.`,
 				DefaultText: "0755",
 				Value:       0755,
 			},
+			&cli.BoolFlag{
+				Name:        "disable-noarchive",
+				Usage:       "disable setting noarchive for multipart part uploads",
+				EnvVars:     []string{"VGW_DISABLE_NOARCHIVE"},
+				Destination: &disableNoArchive,
+			},
 		},
 	}
 }
@@ -98,6 +105,7 @@ func runScoutfs(ctx *cli.Context) error {
 	opts.ChownGID = chowngid
 	opts.BucketLinks = bucketlinks
 	opts.NewDirPerm = fs.FileMode(dirPerms)
+	opts.DisableNoArchive = disableNoArchive
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {