From f0858a47d5ba17aff94f2a88db5f0378044fd45f Mon Sep 17 00:00:00 2001 From: Ondrej Palkovsky Date: Fri, 8 Aug 2025 08:56:44 +0200 Subject: [PATCH] Small cleanups. --- backend/posix/posix.go | 50 +++++++++++++++++++------------------- backend/scoutfs/scoutfs.go | 46 ++++++++++++++++++----------------- 2 files changed, 49 insertions(+), 47 deletions(-) diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 6163de4..22b81b7 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -83,8 +83,8 @@ type Posix struct { var _ backend.Backend = &Posix{} const ( - metaTmpDir = ".sgwtmp" - metaTmpMultipartDir = metaTmpDir + "/multipart" + MetaTmpDir = ".sgwtmp" + MetaTmpMultipartDir = MetaTmpDir + "/multipart" onameAttr = "objname" tagHdr = "X-Amz-Tagging" metaHdr = "X-Amz-Meta" @@ -433,7 +433,7 @@ func (p *Posix) isBucketEmpty(bucket string) error { return fmt.Errorf("readdir bucket: %w", err) } if err == nil { - if len(ents) == 1 && ents[0].Name() != metaTmpDir { + if len(ents) == 1 && ents[0].Name() != MetaTmpDir { return s3err.GetAPIError(s3err.ErrVersionedBucketNotEmpty) } else if len(ents) > 1 { return s3err.GetAPIError(s3err.ErrVersionedBucketNotEmpty) @@ -448,7 +448,7 @@ func (p *Posix) isBucketEmpty(bucket string) error { if errors.Is(err, fs.ErrNotExist) { return s3err.GetAPIError(s3err.ErrNoSuchBucket) } - if len(ents) == 1 && ents[0].Name() != metaTmpDir { + if len(ents) == 1 && ents[0].Name() != MetaTmpDir { return s3err.GetAPIError(s3err.ErrBucketNotEmpty) } else if len(ents) > 1 { return s3err.GetAPIError(s3err.ErrBucketNotEmpty) @@ -694,7 +694,7 @@ func (p *Posix) createObjVersion(bucket, key string, size int64, acc auth.Accoun versionBucketPath := filepath.Join(p.versioningDir, bucket) versioningKey := filepath.Join(genObjVersionKey(key), versionId) - versionTmpPath := filepath.Join(versionBucketPath, metaTmpDir) + versionTmpPath := filepath.Join(versionBucketPath, MetaTmpDir) f, err := p.openTmpFile(versionTmpPath, versionBucketPath, versioningKey, size, acc, doFalloc, p.forceNoTmpFile) if err != nil { @@ -765,7 +765,7 @@ func (p *Posix) ListObjectVersions(ctx context.Context, input *s3.ListObjectVers fileSystem := os.DirFS(bucket) results, err := backend.WalkVersions(ctx, fileSystem, prefix, delim, keyMarker, versionIdMarker, max, - p.fileToObjVersions(bucket), []string{metaTmpDir}) + p.fileToObjVersions(bucket), []string{MetaTmpDir}) if err != nil { return s3response.ListVersionsResult{}, fmt.Errorf("walk %v: %w", bucket, err) } @@ -1211,7 +1211,7 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu s3response.Create objNameSum := sha256.Sum256([]byte(*mpu.Key)) // multiple uploads for same object name allowed, // they will all go into the same hashed name directory - objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", objNameSum)) + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", objNameSum)) tmppath := filepath.Join(bucket, objdir) // the unique upload id is a directory for all of the parts // associated with this specific multipart upload @@ -1400,7 +1400,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C return res, "", err } - objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) checksums, err := p.retrieveChecksums(nil, bucket, filepath.Join(objdir, uploadID)) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { @@ -1502,7 +1502,7 @@ func (p *Posix) CompleteMultipartUploadWithCopy(ctx context.Context, input *s3.C } } - f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, + f, err := p.openTmpFile(filepath.Join(bucket, MetaTmpDir), bucket, object, totalsize, acct, skipFalloc, p.forceNoTmpFile) if err != nil { if errors.Is(err, syscall.EDQUOT) { @@ -1838,7 +1838,7 @@ func numberOfChecksums(part types.CompletedPart) int { func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, error) { sum := sha256.Sum256([]byte(object)) - objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + objdir := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) _, err := os.Stat(filepath.Join(objdir, uploadID)) if errors.Is(err, fs.ErrNotExist) { @@ -1852,7 +1852,7 @@ func (p *Posix) checkUploadIDExists(bucket, object, uploadID string) ([32]byte, func (p *Posix) retrieveUploadId(bucket, object string) (string, [32]byte, error) { sum := sha256.Sum256([]byte(object)) - objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + objdir := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) entries, err := os.ReadDir(objdir) if err != nil || len(entries) == 0 { @@ -2004,7 +2004,7 @@ func (p *Posix) AbortMultipartUpload(_ context.Context, mpu *s3.AbortMultipartUp } sum := sha256.Sum256([]byte(object)) - objdir := filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + objdir := filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) _, err = os.Stat(filepath.Join(objdir, uploadID)) if err != nil { @@ -2042,7 +2042,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl } // ignore readdir error and use the empty list returned - objs, _ := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir)) + objs, _ := os.ReadDir(filepath.Join(bucket, MetaTmpMultipartDir)) var uploads []s3response.Upload var resultUpds []s3response.Upload @@ -2062,7 +2062,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl continue } - b, err := p.meta.RetrieveAttribute(nil, bucket, filepath.Join(metaTmpMultipartDir, obj.Name()), onameAttr) + b, err := p.meta.RetrieveAttribute(nil, bucket, filepath.Join(MetaTmpMultipartDir, obj.Name()), onameAttr) if err != nil { continue } @@ -2071,7 +2071,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl continue } - upids, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, obj.Name())) + upids, err := os.ReadDir(filepath.Join(bucket, MetaTmpMultipartDir, obj.Name())) if err != nil { continue } @@ -2098,7 +2098,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl keyMarkerInd = len(uploads) } - checksum, err := p.retrieveChecksums(nil, bucket, filepath.Join(metaTmpMultipartDir, obj.Name(), uploadID)) + checksum, err := p.retrieveChecksums(nil, bucket, filepath.Join(MetaTmpMultipartDir, obj.Name(), uploadID)) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return lmu, fmt.Errorf("get mp checksum: %w", err) } @@ -2214,7 +2214,7 @@ func (p *Posix) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3resp return lpr, err } - objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) tmpdir := filepath.Join(bucket, objdir) ents, err := os.ReadDir(filepath.Join(tmpdir, uploadID)) @@ -2351,7 +2351,7 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3. } sum := sha256.Sum256([]byte(object)) - objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) mpPath := filepath.Join(objdir, uploadID) _, err = os.Stat(filepath.Join(bucket, mpPath)) @@ -2525,7 +2525,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) } sum := sha256.Sum256([]byte(*upi.Key)) - objdir := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum)) + objdir := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum)) _, err = os.Stat(filepath.Join(*upi.Bucket, objdir, *upi.UploadId)) if errors.Is(err, fs.ErrNotExist) { @@ -2835,7 +2835,7 @@ func (p *Posix) PutObject(ctx context.Context, po s3response.PutObjectInput) (s3 return s3response.PutObjectOutput{}, fmt.Errorf("stat object: %w", err) } - f, err := p.openTmpFile(filepath.Join(*po.Bucket, metaTmpDir), + f, err := p.openTmpFile(filepath.Join(*po.Bucket, MetaTmpDir), *po.Bucket, *po.Key, contentLength, acct, doFalloc, p.forceNoTmpFile) if err != nil { if errors.Is(err, syscall.EDQUOT) { @@ -3189,7 +3189,7 @@ func (p *Posix) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) ( acct = auth.Account{} } - f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), + f, err := p.openTmpFile(filepath.Join(bucket, MetaTmpDir), bucket, object, srcObjVersion.Size(), acct, doFalloc, p.forceNoTmpFile) if err != nil { @@ -3646,7 +3646,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. return nil, err } - ents, err := os.ReadDir(filepath.Join(bucket, metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId)) + ents, err := os.ReadDir(filepath.Join(bucket, MetaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId)) if errors.Is(err, fs.ErrNotExist) { return nil, s3err.GetAPIError(s3err.ErrNoSuchKey) } @@ -3654,7 +3654,7 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. return nil, fmt.Errorf("read parts: %w", err) } - partPath := filepath.Join(metaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber)) + partPath := filepath.Join(MetaTmpMultipartDir, fmt.Sprintf("%x", sum), uploadId, fmt.Sprintf("%v", *input.PartNumber)) part, err := os.Stat(filepath.Join(bucket, partPath)) if errors.Is(err, fs.ErrNotExist) { @@ -4238,7 +4238,7 @@ func (p *Posix) ListObjectsParametrized(ctx context.Context, input *s3.ListObjec fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, - customFileToObj(bucket, true), []string{metaTmpDir}) + customFileToObj(bucket, true), []string{MetaTmpDir}) if err != nil { return s3response.ListObjectsResult{}, fmt.Errorf("walk %v: %w", bucket, err) } @@ -4399,7 +4399,7 @@ func (p *Posix) ListObjectsV2Parametrized(ctx context.Context, input *s3.ListObj fileSystem := os.DirFS(bucket) results, err := backend.Walk(ctx, fileSystem, prefix, delim, marker, maxkeys, - customFileToObj(bucket, fetchOwner), []string{metaTmpDir}) + customFileToObj(bucket, fetchOwner), []string{MetaTmpDir}) if err != nil { return s3response.ListObjectsV2Result{}, fmt.Errorf("walk %v: %w", bucket, err) } diff --git a/backend/scoutfs/scoutfs.go b/backend/scoutfs/scoutfs.go index 3293c17..aac820b 100644 --- a/backend/scoutfs/scoutfs.go +++ b/backend/scoutfs/scoutfs.go @@ -69,11 +69,6 @@ type ScoutFS struct { var _ backend.Backend = &ScoutFS{} const ( - metaTmpDir = ".sgwtmp" - metaTmpMultipartDir = metaTmpDir + "/multipart" -) - -var ( stageComplete = "ongoing-request=\"false\", expiry-date=\"Fri, 2 Dec 2050 00:00:00 GMT\"" stageInProgress = "true" stageNotInProgress = "false" @@ -123,7 +118,7 @@ func (s *ScoutFS) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s sum := sha256.Sum256([]byte(*input.Key)) partPath := filepath.Join( *input.Bucket, // bucket - metaTmpMultipartDir, // temp multipart dir + posix.MetaTmpMultipartDir, // temp multipart dir fmt.Sprintf("%x", sum), // hashed objname *input.UploadId, // upload id fmt.Sprintf("%v", *input.PartNumber), // part number @@ -239,14 +234,23 @@ func (s *ScoutFS) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3. } func (s *ScoutFS) ListObjects(ctx context.Context, input *s3.ListObjectsInput) (s3response.ListObjectsResult, error) { - return s.Posix.ListObjectsParametrized(ctx, input, s.fileToObj) + if s.glaciermode { + return s.Posix.ListObjectsParametrized(ctx, input, s.glacierFileToObj) + } else { + return s.Posix.ListObjects(ctx, input) + } } func (s *ScoutFS) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input) (s3response.ListObjectsV2Result, error) { - return s.Posix.ListObjectsV2Parametrized(ctx, input, s.fileToObj) + if s.glaciermode { + return s.Posix.ListObjectsV2Parametrized(ctx, input, s.glacierFileToObj) + } else { + return s.Posix.ListObjectsV2(ctx, input) + } } -func (s *ScoutFS) fileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { +// FileToObj function for ListObject calls that adds a Glacier storage class if the file is offline +func (s *ScoutFS) glacierFileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { posixFileToObj := s.Posix.FileToObj(bucket, fetchOwner) return func(path string, d fs.DirEntry) (s3response.Object, error) { @@ -255,19 +259,17 @@ func (s *ScoutFS) fileToObj(bucket string, fetchOwner bool) backend.GetObjFunc { return res, err } objPath := filepath.Join(bucket, path) - if s.glaciermode { - // Check if there are any offline exents associated with this file. - // If so, we will return the Glacier storage class - st, err := statMore(objPath) - if errors.Is(err, fs.ErrNotExist) { - return s3response.Object{}, backend.ErrSkipObj - } - if err != nil { - return s3response.Object{}, fmt.Errorf("stat more: %w", err) - } - if st.Offline_blocks != 0 { - res.StorageClass = types.ObjectStorageClassGlacier - } + // Check if there are any offline exents associated with this file. + // If so, we will return the Glacier storage class + st, err := statMore(objPath) + if errors.Is(err, fs.ErrNotExist) { + return s3response.Object{}, backend.ErrSkipObj + } + if err != nil { + return s3response.Object{}, fmt.Errorf("stat more: %w", err) + } + if st.Offline_blocks != 0 { + res.StorageClass = types.ObjectStorageClassGlacier } return res, nil }