feat: add configurable default ETag for files without metadata

When versitygw serves files that were placed on the filesystem outside
of the gateway, those files have no stored etag attribute and are
served with an empty ETag. This causes problems for S3 clients that
rely on ETags being non empty.

The new --default-etag flag (VGW_DEFAULT_ETAG) lets operators specify
a fallback ETag value returned for any object that lacks a stored etag
attribute, making pre-existing filesystem data behave more predictably
under S3 workloads without requiring a metadata migration.

backend/posix/posix.go

@@ -91,6 +91,10 @@ type Posix struct {
 	// frontend handlers are already validating bucket names
 	validateBucketName bool
 
+	// defaultEtag is returned for objects that do not have an etag attribute
+	// stored (e.g. files placed on the filesystem outside of versitygw).
+	defaultEtag string
+
 	// actionLimiter limits the number of concurrently running POSIX actions.
 	// The primary goal is to bound OS thread growth: when goroutines block in
 	// filesystem syscalls, the Go scheduler may spawn additional OS threads to
@@ -186,6 +190,10 @@ type PosixOpts struct {
 	// threshold are rejected with an 'InvalidRequest' error. Defaults to the
 	// S3 specification limit of 5 GiB.
 	CopyObjectThreshold int64
+	// DefaultEtag is returned for objects that do not have a stored etag
+	// attribute (e.g. files placed on the filesystem outside of versitygw).
+	// When empty, such objects are served with an empty ETag.
+	DefaultEtag string
 }
 
 func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) {
@@ -250,6 +258,7 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro
 		validateBucketName:   opts.ValidateBucketNames,
 		actionLimiter:        semaphore.NewWeighted(int64(concurrencyOrDefault(opts.Concurrency))),
 		copyObjectThreshold:  opts.CopyObjectThreshold,
+		defaultEtag:          opts.DefaultEtag,
 	}, nil
 }
 
@@ -3391,8 +3400,8 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput)
 	// evaluate preconditions
 	b, err := p.meta.RetrieveAttribute(srcf, srcBucket, srcObject, etagkey)
 	srcEtag := string(b)
-	if err != nil {
-		srcEtag = ""
+	if err != nil || srcEtag == "" {
+		srcEtag = p.defaultEtag
 	}
 
 	err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{
@@ -4057,8 +4066,8 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (
 
 		b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 		etag := string(b)
-		if err != nil {
-			etag = ""
+		if err != nil || etag == "" {
+			etag = p.defaultEtag
 		}
 
 		// evaluate preconditions
@@ -4575,8 +4584,8 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge
 
 	b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	etag := string(b)
-	if err != nil {
-		etag = ""
+	if err != nil || etag == "" {
+		etag = p.defaultEtag
 	}
 
 	// evaluate preconditions
@@ -4928,8 +4937,8 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.
 
 	b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey)
 	etag := string(b)
-	if err != nil {
-		etag = ""
+	if err != nil || etag == "" {
+		etag = p.defaultEtag
 	}
 
 	// evaluate preconditions
@@ -5242,8 +5251,8 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput
 
 	b, err := p.meta.RetrieveAttribute(f, srcBucket, srcObject, etagkey)
 	srcEtag := string(b)
-	if err != nil {
-		srcEtag = ""
+	if err != nil || srcEtag == "" {
+		srcEtag = p.defaultEtag
 	}
 
 	err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{
@@ -5654,6 +5663,9 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc {
 		// so this will just set etag to "" if its not already set
 
 		etag := string(etagBytes)
+		if etag == "" {
+			etag = p.defaultEtag
+		}
 
 		fi, err := d.Info()
 		if errors.Is(err, fs.ErrNotExist) {

backend/scoutfs/scoutfs.go

@@ -50,6 +50,10 @@ type ScoutfsOpts struct {
 	// threshold are rejected with an 'InvalidRequest' error. Defaults to the
 	// S3 specification limit of 5 GiB.
 	CopyObjectThreshold int64
+	// DefaultEtag is returned for objects that do not have a stored etag
+	// attribute (e.g. files placed on the filesystem outside of versitygw).
+	// When empty, such objects are served with an empty ETag.
+	DefaultEtag string
 }
 
 var _ backend.Backend = &ScoutFS{}

backend/scoutfs/scoutfs_compat.go

@@ -83,6 +83,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) {
 		ValidateBucketNames: opts.ValidateBucketNames,
 		Concurrency:         opts.Concurrency,
 		CopyObjectThreshold: opts.CopyObjectThreshold,
+		DefaultEtag:         opts.DefaultEtag,
 	})
 	if err != nil {
 		return nil, err

cmd/versitygw/posix.go

@@ -34,6 +34,7 @@ var (
 	forceNoTmpFile       bool
 	forceNoCopyFileRange bool
 	actionsConcurrency   int
+	defaultEtag          string
 )
 
 func posixCommand() *cli.Command {
@@ -115,6 +116,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`,
 				EnvVars:     []string{"VGW_DISABLE_COPY_FILE_RANGE"},
 				Destination: &forceNoCopyFileRange,
 			},
+			&cli.StringFlag{
+				Name:        "default-etag",
+				Usage:       "default ETag value returned for objects that do not have a stored etag attribute (e.g. files placed on the filesystem outside of versitygw)",
+				EnvVars:     []string{"VGW_DEFAULT_ETAG"},
+				Destination: &defaultEtag,
+			},
 		},
 	}
 }
@@ -149,6 +156,7 @@ func runPosix(ctx *cli.Context) error {
 		ValidateBucketNames:  disableStrictBucketNames,
 		Concurrency:          actionsConcurrency,
 		CopyObjectThreshold:  copyObjectThreshold,
+		DefaultEtag:          defaultEtag,
 	}
 
 	var ms meta.MetadataStorer

cmd/versitygw/scoutfs.go

@@ -106,6 +106,12 @@ move interfaces as well as support for tiered filesystems.`,
 				Value:       5000,
 				Destination: &actionsConcurrency,
 			},
+			&cli.StringFlag{
+				Name:        "default-etag",
+				Usage:       "default ETag value returned for objects that do not have a stored etag attribute (e.g. files placed on the filesystem outside of versitygw)",
+				EnvVars:     []string{"VGW_DEFAULT_ETAG"},
+				Destination: &defaultEtag,
+			},
 		},
 	}
 }
@@ -135,6 +141,7 @@ func runScoutfs(ctx *cli.Context) error {
 	opts.SetProjectID = setProjectID
 	opts.Concurrency = actionsConcurrency
 	opts.CopyObjectThreshold = copyObjectThreshold
+	opts.DefaultEtag = defaultEtag
 
 	be, err := scoutfs.New(ctx.Args().Get(0), opts)
 	if err != nil {

extra/example.conf

@@ -572,6 +572,16 @@ ROOT_SECRET_ACCESS_KEY=
 # than O_TMPFILE when the data needs to be copied into the final location.
 #VGW_DISABLE_OTMP=false
 
+# The VGW_DEFAULT_ETAG option sets the ETag value returned for objects that do
+# not have a stored etag attribute. This applies to files that were placed on
+# the filesystem outside of versitygw and therefore lack S3 metadata. Some S3
+# clients assume a plain ETag is an MD5 checksum unless it contains a "-", as
+# multipart upload ETags use the format <hash>-<parts>. To avoid clients
+# incorrectly treating this default value as an MD5 checksum, prefer a value
+# that includes a "-" and is not just 32 hex characters, for example:
+# unknown-1. When not set, such objects are served with an empty ETag.
+#VGW_DEFAULT_ETAG=
+
 ###########
 # scoutfs #
 ###########
@@ -633,6 +643,16 @@ ROOT_SECRET_ACCESS_KEY=
 # VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior.
 #VGW_DISABLE_NOARCHIVE=false
 
+# The VGW_DEFAULT_ETAG option sets the ETag value returned for objects that do
+# not have a stored etag attribute. This applies to files that were placed on
+# the filesystem outside of versitygw and therefore lack S3 metadata. Some S3
+# clients assume a plain ETag is an MD5 checksum unless it contains a "-", as
+# multipart upload ETags use the format <hash>-<parts>. To avoid clients
+# incorrectly treating this default value as an MD5 checksum, prefer a value
+# that includes a "-" and is not just 32 hex characters, for example:
+# unknown-1. When not set, such objects are served with an empty ETag.
+#VGW_DEFAULT_ETAG=
+
 ######
 # s3 #
 ######