Merge pull request #2218 from versity/ben/default-etag

feat: add configurable default ETag for files without metadata

backend/posix/posix.go

@@ -91,6 +91,10 @@ type Posix struct {
 	// frontend handlers are already validating bucket names
 	validateBucketName bool
 
+	// defaultEtag is returned for objects that do not have an etag attribute
+	// stored (e.g. files placed on the filesystem outside of versitygw).
+	defaultEtag string
+
 	// actionLimiter limits the number of concurrently running POSIX actions.
 	// The primary goal is to bound OS thread growth: when goroutines block in
 	// filesystem syscalls, the Go scheduler may spawn additional OS threads to
@@ -186,6 +190,10 @@ type PosixOpts struct {
 	// threshold are rejected with an 'InvalidRequest' error. Defaults to the
 	// S3 specification limit of 5 GiB.
 	CopyObjectThreshold int64
+	// DefaultEtag is returned for objects that do not have a stored etag
+	// attribute (e.g. files placed on the filesystem outside of versitygw).
+	// When empty, such objects are served with an empty ETag.
+	DefaultEtag string
 }
 
 func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) {
@@ -250,6 +258,7 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
 		validateBucketName:   opts.ValidateBucketNames,
 		actionLimiter:        semaphore.NewWeighted(int64(concurrencyOrDefault(opts.Concurrency))),
 		copyObjectThreshold:  opts.CopyObjectThreshold,
+		defaultEtag:          opts.DefaultEtag,
 	}, nil
 }
 
@@ -3391,8 +3400,8 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
 	// evaluate preconditions
 	b, err := p.meta.RetrieveAttribute(srcf, srcBucket, srcObject, etagkey)
 	srcEtag := string(b)
-	if err != nil {
-		srcEtag = ""
+	if err != nil || srcEtag == "" {
+		srcEtag = p.defaultEtag
 	}
 
 	err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{
@@ -4057,8 +4066,8 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
 
 		b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 		etag := string(b)
-		if err != nil {
-			etag = ""
+		if err != nil || etag == "" {
+			etag = p.defaultEtag
 		}
 
 		// evaluate preconditions
@@ -4575,8 +4584,8 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
 
 	b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	etag := string(b)
-	if err != nil {
-		etag = ""
+	if err != nil || etag == "" {
+		etag = p.defaultEtag
 	}
 
 	// evaluate preconditions
@@ -4928,8 +4937,8 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
 
 	b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	etag := string(b)
-	if err != nil {
-		etag = ""
+	if err != nil || etag == "" {
+		etag = p.defaultEtag
 	}
 
 	// evaluate preconditions
@@ -5242,8 +5251,8 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
 
 	b, err := p.meta.RetrieveAttribute(f, srcBucket, srcObject, etagkey)
 	srcEtag := string(b)
-	if err != nil {
-		srcEtag = ""
+	if err != nil || srcEtag == "" {
+		srcEtag = p.defaultEtag
 	}
 
 	err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{
@@ -5654,6 +5663,9 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc {
 		// so this will just set etag to "" if its not already set
 
 		etag := string(etagBytes)
+		if etag == "" {
+			etag = p.defaultEtag
+		}
 
 		fi, err := d.Info()
 		if errors.Is(err, fs.ErrNotExist) {

backend/scoutfs/scoutfs.go

@@ -50,6 +50,10 @@ type ScoutfsOpts struct {
 	// threshold are rejected with an 'InvalidRequest' error. Defaults to the
 	// S3 specification limit of 5 GiB.
 	CopyObjectThreshold int64
+	// DefaultEtag is returned for objects that do not have a stored etag
+	// attribute (e.g. files placed on the filesystem outside of versitygw).
+	// When empty, such objects are served with an empty ETag.
+	DefaultEtag string
 }
 
 var _ backend.Backend = &ScoutFS{}

backend/scoutfs/scoutfs_compat.go

@@ -83,6 +83,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 		ValidateBucketNames: opts.ValidateBucketNames,
 		Concurrency:         opts.Concurrency,
 		CopyObjectThreshold: opts.CopyObjectThreshold,
+		DefaultEtag:         opts.DefaultEtag,
 	})
 	if err != nil {
 		return nil, err

cmd/versitygw/posix.go

@@ -34,6 +34,7 @@ var (
 	forceNoTmpFile       bool
 	forceNoCopyFileRange bool
 	actionsConcurrency   int
+	defaultEtag          string
 )
 
 func posixCommand() *cli.Command {
@@ -115,6 +116,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_DISABLE_COPY_FILE_RANGE"},
 				Destination: &forceNoCopyFileRange,
 			},
+			&cli.StringFlag{
+				Name:        "default-etag",
+				Usage:       "default ETag value returned for objects that do not have a stored etag attribute (e.g. files placed on the filesystem outside of versitygw)",
+				EnvVars:     []string{"VGW_DEFAULT_ETAG"},
+				Destination: &defaultEtag,
+			},
 		},
 	}
 }
@@ -149,6 +156,7 @@ func runPosix(ctx *cli.Context) error {
 		ValidateBucketNames:  disableStrictBucketNames,
 		Concurrency:          actionsConcurrency,
 		CopyObjectThreshold:  copyObjectThreshold,
+		DefaultEtag:          defaultEtag,
 	}
 
 	var ms meta.MetadataStorer

cmd/versitygw/scoutfs.go

@@ -106,6 +106,12 @@ move interfaces as well as support for tiered filesystems.`,
 				Value:       5000,
 				Destination: &actionsConcurrency,
 			},
+			&cli.StringFlag{
+				Name:        "default-etag",
+				Usage:       "default ETag value returned for objects that do not have a stored etag attribute (e.g. files placed on the filesystem outside of versitygw)",
+				EnvVars:     []string{"VGW_DEFAULT_ETAG"},
+				Destination: &defaultEtag,
+			},
 		},
 	}
 }
@@ -135,6 +141,7 @@ func runScoutfs(ctx *cli.Context) error {
 	opts.SetProjectID = setProjectID
 	opts.Concurrency = actionsConcurrency
 	opts.CopyObjectThreshold = copyObjectThreshold
+	opts.DefaultEtag = defaultEtag
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

extra/example.conf

@@ -572,6 +572,16 @@ ROOT_SECRET_ACCESS_KEY=
 # than O_TMPFILE when the data needs to be copied into the final location.
 #VGW_DISABLE_OTMP=false
 
+# The VGW_DEFAULT_ETAG option sets the ETag value returned for objects that do
+# not have a stored etag attribute. This applies to files that were placed on
+# the filesystem outside of versitygw and therefore lack S3 metadata. Some S3
+# clients assume a plain ETag is an MD5 checksum unless it contains a "-", as
+# multipart upload ETags use the format <hash>-<parts>. To avoid clients
+# incorrectly treating this default value as an MD5 checksum, prefer a value
+# that includes a "-" and is not just 32 hex characters, for example:
+# unknown-1. When not set, such objects are served with an empty ETag.
+#VGW_DEFAULT_ETAG=
+
 ###########
 # scoutfs #
 ###########
@@ -633,6 +643,16 @@ ROOT_SECRET_ACCESS_KEY=
 # VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior.
 #VGW_DISABLE_NOARCHIVE=false
 
+# The VGW_DEFAULT_ETAG option sets the ETag value returned for objects that do
+# not have a stored etag attribute. This applies to files that were placed on
+# the filesystem outside of versitygw and therefore lack S3 metadata. Some S3
+# clients assume a plain ETag is an MD5 checksum unless it contains a "-", as
+# multipart upload ETags use the format <hash>-<parts>. To avoid clients
+# incorrectly treating this default value as an MD5 checksum, prefer a value
+# that includes a "-" and is not just 32 hex characters, for example:
+# unknown-1. When not set, such objects are served with an empty ETag.
+#VGW_DEFAULT_ETAG=
+
 ######
 # s3 #
 ######