diff --git a/backend/posix/posix.go b/backend/posix/posix.go index bf6e1a8e..bf77d14e 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -91,6 +91,10 @@ type Posix struct { // frontend handlers are already validating bucket names validateBucketName bool + // defaultEtag is returned for objects that do not have an etag attribute + // stored (e.g. files placed on the filesystem outside of versitygw). + defaultEtag string + // actionLimiter limits the number of concurrently running POSIX actions. // The primary goal is to bound OS thread growth: when goroutines block in // filesystem syscalls, the Go scheduler may spawn additional OS threads to @@ -186,6 +190,10 @@ type PosixOpts struct { // threshold are rejected with an 'InvalidRequest' error. Defaults to the // S3 specification limit of 5 GiB. CopyObjectThreshold int64 + // DefaultEtag is returned for objects that do not have a stored etag + // attribute (e.g. files placed on the filesystem outside of versitygw). + // When empty, such objects are served with an empty ETag. + DefaultEtag string } func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, error) { @@ -250,6 +258,7 @@ func New(rootdir string, meta meta.MetadataStorer, opts PosixOpts) (*Posix, erro validateBucketName: opts.ValidateBucketNames, actionLimiter: semaphore.NewWeighted(int64(concurrencyOrDefault(opts.Concurrency))), copyObjectThreshold: opts.CopyObjectThreshold, + defaultEtag: opts.DefaultEtag, }, nil } @@ -3391,8 +3400,8 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) // evaluate preconditions b, err := p.meta.RetrieveAttribute(srcf, srcBucket, srcObject, etagkey) srcEtag := string(b) - if err != nil { - srcEtag = "" + if err != nil || srcEtag == "" { + srcEtag = p.defaultEtag } err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{ @@ -4057,8 +4066,8 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) etag := string(b) - if err != nil { - etag = "" + if err != nil || etag == "" { + etag = p.defaultEtag } // evaluate preconditions @@ -4575,8 +4584,8 @@ func (p *Posix) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.Ge b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) etag := string(b) - if err != nil { - etag = "" + if err != nil || etag == "" { + etag = p.defaultEtag } // evaluate preconditions @@ -4928,8 +4937,8 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. b, err := p.meta.RetrieveAttribute(nil, bucket, object, etagkey) etag := string(b) - if err != nil { - etag = "" + if err != nil || etag == "" { + etag = p.defaultEtag } // evaluate preconditions @@ -5242,8 +5251,8 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput b, err := p.meta.RetrieveAttribute(f, srcBucket, srcObject, etagkey) srcEtag := string(b) - if err != nil { - srcEtag = "" + if err != nil || srcEtag == "" { + srcEtag = p.defaultEtag } err = backend.EvaluatePreconditions(srcEtag, fi.ModTime(), backend.PreConditions{ @@ -5654,6 +5663,9 @@ func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { // so this will just set etag to "" if its not already set etag := string(etagBytes) + if etag == "" { + etag = p.defaultEtag + } fi, err := d.Info() if errors.Is(err, fs.ErrNotExist) { diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 5f9e6eec..98836b12 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -50,6 +50,10 @@ type ScoutfsOpts struct { // threshold are rejected with an 'InvalidRequest' error. Defaults to the // S3 specification limit of 5 GiB. CopyObjectThreshold int64 + // DefaultEtag is returned for objects that do not have a stored etag + // attribute (e.g. files placed on the filesystem outside of versitygw). + // When empty, such objects are served with an empty ETag. + DefaultEtag string } var _ backend.Backend = &ScoutFS{} diff --git a/backend/scoutfs/scoutfs_compat.go b/backend/scoutfs/scoutfs_compat.go index 7b242f7b..15076417 100644 --- a/backend/scoutfs/scoutfs_compat.go +++ b/backend/scoutfs/scoutfs_compat.go @@ -83,6 +83,7 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) { ValidateBucketNames: opts.ValidateBucketNames, Concurrency: opts.Concurrency, CopyObjectThreshold: opts.CopyObjectThreshold, + DefaultEtag: opts.DefaultEtag, }) if err != nil { return nil, err diff --git a/cmd/versitygw/posix.go b/cmd/versitygw/posix.go index 913a0ae6..c1541edd 100644 --- a/cmd/versitygw/posix.go +++ b/cmd/versitygw/posix.go @@ -34,6 +34,7 @@ var ( forceNoTmpFile bool forceNoCopyFileRange bool actionsConcurrency int + defaultEtag string ) func posixCommand() *cli.Command { @@ -115,6 +116,12 @@ will be translated into the file /mnt/fs/gwroot/mybucket/a/b/c/myobject`, EnvVars: []string{"VGW_DISABLE_COPY_FILE_RANGE"}, Destination: &forceNoCopyFileRange, }, + &cli.StringFlag{ + Name: "default-etag", + Usage: "default ETag value returned for objects that do not have a stored etag attribute (e.g. files placed on the filesystem outside of versitygw)", + EnvVars: []string{"VGW_DEFAULT_ETAG"}, + Destination: &defaultEtag, + }, }, } } @@ -149,6 +156,7 @@ func runPosix(ctx *cli.Context) error { ValidateBucketNames: disableStrictBucketNames, Concurrency: actionsConcurrency, CopyObjectThreshold: copyObjectThreshold, + DefaultEtag: defaultEtag, } var ms meta.MetadataStorer diff --git a/cmd/versitygw/scoutfs.go b/cmd/versitygw/scoutfs.go index 91c2909a..bd0c012f 100644 --- a/cmd/versitygw/scoutfs.go +++ b/cmd/versitygw/scoutfs.go @@ -106,6 +106,12 @@ move interfaces as well as support for tiered filesystems.`, Value: 5000, Destination: &actionsConcurrency, }, + &cli.StringFlag{ + Name: "default-etag", + Usage: "default ETag value returned for objects that do not have a stored etag attribute (e.g. files placed on the filesystem outside of versitygw)", + EnvVars: []string{"VGW_DEFAULT_ETAG"}, + Destination: &defaultEtag, + }, }, } } @@ -135,6 +141,7 @@ func runScoutfs(ctx *cli.Context) error { opts.SetProjectID = setProjectID opts.Concurrency = actionsConcurrency opts.CopyObjectThreshold = copyObjectThreshold + opts.DefaultEtag = defaultEtag be, err := scoutfs.New(ctx.Args().Get(0), opts) if err != nil { diff --git a/extra/example.conf b/extra/example.conf index 8f2f6c1d..86c4d5db 100644 --- a/extra/example.conf +++ b/extra/example.conf @@ -572,6 +572,16 @@ ROOT_SECRET_ACCESS_KEY= # than O_TMPFILE when the data needs to be copied into the final location. #VGW_DISABLE_OTMP=false +# The VGW_DEFAULT_ETAG option sets the ETag value returned for objects that do +# not have a stored etag attribute. This applies to files that were placed on +# the filesystem outside of versitygw and therefore lack S3 metadata. Some S3 +# clients assume a plain ETag is an MD5 checksum unless it contains a "-", as +# multipart upload ETags use the format -. To avoid clients +# incorrectly treating this default value as an MD5 checksum, prefer a value +# that includes a "-" and is not just 32 hex characters, for example: +# unknown-1. When not set, such objects are served with an empty ETag. +#VGW_DEFAULT_ETAG= + ########### # scoutfs # ########### @@ -633,6 +643,16 @@ ROOT_SECRET_ACCESS_KEY= # VGW_DISABLE_NOARCHIVE option can be set to true to disable this behavior. #VGW_DISABLE_NOARCHIVE=false +# The VGW_DEFAULT_ETAG option sets the ETag value returned for objects that do +# not have a stored etag attribute. This applies to files that were placed on +# the filesystem outside of versitygw and therefore lack S3 metadata. Some S3 +# clients assume a plain ETag is an MD5 checksum unless it contains a "-", as +# multipart upload ETags use the format -. To avoid clients +# incorrectly treating this default value as an MD5 checksum, prefer a value +# that includes a "-" and is not just 32 hex characters, for example: +# unknown-1. When not set, such objects are served with an empty ETag. +#VGW_DEFAULT_ETAG= + ###### # s3 # ######