diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 4244d72..6163de4 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -4206,6 +4206,10 @@ func (p *Posix) CopyObject(ctx context.Context, input s3response.CopyObjectInput } func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { + return p.ListObjectsParametrized(ctx, input, p.FileToObj) +} + +func (p *Posix) ListObjectsParametrized(ctx context.Context, input *s3.ListObjectsInput, customFileToObj func(string, bool) backend.GetObjFunc) (s3response.ListObjectsResult, error) { bucket := *input.Bucket prefix := "" if input.Prefix != nil { @@ -4234,7 +4238,7 @@ func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3 fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, - p.fileToObj(bucket, true), []string{metaTmpDir}) + customFileToObj(bucket, true), []string{metaTmpDir}) if err != nil { return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err) } @@ -4252,7 +4256,7 @@ func (p *Posix) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3 }, nil } -func (p *Posix) fileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { +func (p *Posix) FileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { return func(path string, d fs.DirEntry) (s3response.Object, error) { var owner *types.Owner // Retreive the object owner data from bucket ACL, if fetchOwner is true @@ -4355,6 +4359,10 @@ func (p *Posix) fileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { } func (p *Posix) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { + return p.ListObjectsV2Parametrized(ctx, input, p.FileToObj) +} + +func (p *Posix) ListObjectsV2Parametrized(ctx context.Context, input *s3.ListObjectsV2Input, customFileToObj func(string, bool) backend.GetObjFunc) (s3response.ListObjectsV2Result, error) { bucket := *input.Bucket prefix := "" if input.Prefix != nil { @@ -4391,7 +4399,7 @@ func (p *Posix) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, - p.fileToObj(bucket, fetchOwner), []string{metaTmpDir}) + customFileToObj(bucket, fetchOwner), []string{metaTmpDir}) if err != nil { return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err) } diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index df42e5d..3293c17 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -30,7 +30,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/pkg/xattr" "github.com/versity/versitygw/backend" - "github.com/versity/versitygw/backend/meta" "github.com/versity/versitygw/backend/posix" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" @@ -50,9 +49,6 @@ type ScoutFS struct { rootfd *os.File rootdir string - // bucket/object metadata storage facility - meta meta.MetadataStorer - // glaciermode enables the following behavior: // GET object: if file offline, return invalid object state // HEAD object: if file offline, set obj storage class to GLACIER @@ -75,8 +71,6 @@ var _ backend.Backend = &ScoutFS{} const ( metaTmpDir = ".sgwtmp" metaTmpMultipartDir = metaTmpDir + "/multipart" - etagkey = "etag" - checksumsKey = "checksums" ) var ( @@ -245,173 +239,25 @@ func (s *ScoutFS) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3. } func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { - bucket := *input.Bucket - prefix := "" - if input.Prefix != nil { - prefix = *input.Prefix - } - marker := "" - if input.Marker != nil { - marker = *input.Marker - } - delim := "" - if input.Delimiter != nil { - delim = *input.Delimiter - } - maxkeys := int32(0) - if input.MaxKeys != nil { - maxkeys = *input.MaxKeys - } - - _, err := os.Stat(bucket) - if errors.Is(err, fs.ErrNotExist) { - return s3response.ListObjectsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) - } - if err != nil { - return s3response.ListObjectsResult{}, fmt.Errorf("stat bucket: %w", err) - } - - fileSystem := os.DirFS(bucket) - results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, - s.fileToObj(bucket), []string{metaTmpDir}) - if err != nil { - return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err) - } - - return s3response.ListObjectsResult{ - CommonPrefixes: results.CommonPrefixes, - Contents: results.Objects, - Delimiter: backend.GetPtrFromString(delim), - Marker: backend.GetPtrFromString(marker), - NextMarker: backend.GetPtrFromString(results.NextMarker), - Prefix: backend.GetPtrFromString(prefix), - IsTruncated: &results.Truncated, - MaxKeys: &maxkeys, - Name: &bucket, - }, nil + return s.Posix.ListObjectsParametrized(ctx, input, s.fileToObj) } func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { - bucket := *input.Bucket - prefix := "" - if input.Prefix != nil { - prefix = *input.Prefix - } - marker := "" - if input.ContinuationToken != nil { - if input.StartAfter != nil { - marker = max(*input.StartAfter, *input.ContinuationToken) - } else { - marker = *input.ContinuationToken - } - } - delim := "" - if input.Delimiter != nil { - delim = *input.Delimiter - } - maxkeys := int32(0) - if input.MaxKeys != nil { - maxkeys = *input.MaxKeys - } - - _, err := os.Stat(bucket) - if errors.Is(err, fs.ErrNotExist) { - return s3response.ListObjectsV2Result{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) - } - if err != nil { - return s3response.ListObjectsV2Result{}, fmt.Errorf("stat bucket: %w", err) - } - - fileSystem := os.DirFS(bucket) - results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, int32(maxkeys), - s.fileToObj(bucket), []string{metaTmpDir}) - if err != nil { - return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err) - } - - count := int32(len(results.Objects)) - - return s3response.ListObjectsV2Result{ - CommonPrefixes: results.CommonPrefixes, - Contents: results.Objects, - IsTruncated: &results.Truncated, - MaxKeys: &maxkeys, - Name: &bucket, - KeyCount: &count, - Delimiter: backend.GetPtrFromString(delim), - ContinuationToken: backend.GetPtrFromString(marker), - NextContinuationToken: backend.GetPtrFromString(results.NextMarker), - Prefix: backend.GetPtrFromString(prefix), - StartAfter: backend.GetPtrFromString(*input.StartAfter), - }, nil + return s.Posix.ListObjectsV2Parametrized(ctx, input, s.fileToObj) } -func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { +func (s *ScoutFS) fileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { + posixFileToObj := s.Posix.FileToObj(bucket, fetchOwner) + return func(path string, d fs.DirEntry) (s3response.Object, error) { + res, err := posixFileToObj(path, d) + if err != nil || d.IsDir() { + return res, err + } objPath := filepath.Join(bucket, path) - if d.IsDir() { - // directory object only happens if directory empty - // check to see if this is a directory object by checking etag - etagBytes, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey) - if errors.Is(err, meta.ErrNoSuchKey) || errors.Is(err, fs.ErrNotExist) { - return s3response.Object{}, backend.ErrSkipObj - } - if err != nil { - return s3response.Object{}, fmt.Errorf("get etag: %w", err) - } - etag := string(etagBytes) - - fi, err := d.Info() - if errors.Is(err, fs.ErrNotExist) { - return s3response.Object{}, backend.ErrSkipObj - } - if err != nil { - return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err) - } - - size := int64(0) - mtime := fi.ModTime() - - return s3response.Object{ - ETag: &etag, - Key: &path, - LastModified: &mtime, - Size: &size, - StorageClass: types.ObjectStorageClassStandard, - }, nil - } - - // Retreive the object checksum algorithm - checksums, err := s.retrieveChecksums(nil, bucket, path) - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return s3response.Object{}, backend.ErrSkipObj - } - - // file object, get object info and fill out object data - b, err := s.meta.RetrieveAttribute(nil, bucket, path, etagkey) - if errors.Is(err, fs.ErrNotExist) { - return s3response.Object{}, backend.ErrSkipObj - } - if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { - return s3response.Object{}, fmt.Errorf("get etag: %w", err) - } - // note: meta.ErrNoSuchKey will return etagBytes = []byte{} - // so this will just set etag to "" if its not already set - - etag := string(b) - - fi, err := d.Info() - if errors.Is(err, fs.ErrNotExist) { - return s3response.Object{}, backend.ErrSkipObj - } - if err != nil { - return s3response.Object{}, fmt.Errorf("get fileinfo: %w", err) - } - - sc := types.ObjectStorageClassStandard if s.glaciermode { // Check if there are any offline exents associated with this file. - // If so, we will return the InvalidObjectState error. + // If so, we will return the Glacier storage class st, err := statMore(objPath) if errors.Is(err, fs.ErrNotExist) { return s3response.Object{}, backend.ErrSkipObj @@ -420,35 +266,13 @@ func (s *ScoutFS) fileToObj(bucket string) backend.GetObjFunc { return s3response.Object{}, fmt.Errorf("stat more: %w", err) } if st.Offline_blocks != 0 { - sc = types.ObjectStorageClassGlacier + res.StorageClass = types.ObjectStorageClassGlacier } } - - size := fi.Size() - mtime := fi.ModTime() - - return s3response.Object{ - ETag: &etag, - Key: &path, - LastModified: &mtime, - Size: &size, - StorageClass: sc, - ChecksumAlgorithm: []types.ChecksumAlgorithm{checksums.Algorithm}, - ChecksumType: checksums.Type, - }, nil + return res, nil } } -func (s *ScoutFS) retrieveChecksums(f *os.File, bucket, object string) (checksums s3response.Checksum, err error) { - checksumsAtr, err := s.meta.RetrieveAttribute(f, bucket, object, checksumsKey) - if err != nil { - return checksums, err - } - - err = json.Unmarshal(checksumsAtr, &checksums) - return checksums, err -} - // RestoreObject will set stage request on file if offline and do nothing if // file is online func (s *ScoutFS) RestoreObject(_ context.Context, input *s3.RestoreObjectInput) error { diff --git a/backend/scoutfs/scoutfs_compat.go b/backend/scoutfs/scoutfs_compat.go index ec89b3e..4573a7b 100644 --- a/backend/scoutfs/scoutfs_compat.go +++ b/backend/scoutfs/scoutfs_compat.go @@ -48,7 +48,6 @@ func New(rootdir string, opts ScoutfsOpts) (*ScoutFS, error) { Posix: p, rootfd: f, rootdir: rootdir, - meta: metastore, glaciermode: opts.GlacierMode, disableNoArchive: opts.DisableNoArchive, }, nil