mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-16 12:16:36 +00:00
Chris LuandGitHub
68a4e3347f
* s3: track manifest blob ownership through multipart completion Manifest blobs made three orphan paths. A partial fold that failed midway kept its earlier batches on volume servers while the write fell back to flat chunks; the fold now records each saved blob and deletes them on error. A completion that failed after preparing left its fresh manifests behind on every retry; the completion state now owns them and deletes them unless a failed rollback left the version entry still holding them. And a completed upload removes its parts metadata-only, which stranded the part-manifest blobs superseded by flattening; those are collected during flattening and deleted once the completion commits. Two shared-chunk hazards nearby: the version-file rollback deleted its data, destroying the still-registered parts (worse once manifests resolve to inner chunks), and the idempotent-replay cleanup data-deleted leftover parts whose chunks the live object references. Both are metadata-only now. * s3: trim chunk manifest comments * s3: test manifest fold rollback and part range selection The fold-with-rollback and the boundary-to-byte-range logic were only exercised by hand against a live server; give both an injectable seam and cover the fold, the below-threshold and SSE no-ops, the midway failure deleting the blobs it saved, and offset-vs-legacy-index range selection including indexes that no longer address the chunk list.
75 lines
2.7 KiB
Go
75 lines
2.7 KiB
Go
package s3api
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"math"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
)
|
|
|
|
// saveManifestChunk stores manifest blobs, assigning volumes against the
|
|
// object's filer path so bucket placement rules apply.
|
|
func (s3a *S3ApiServer) saveManifestChunk(filePath string, bucket string, ttlSec int32) filer.SaveDataAsChunkFunctionType {
|
|
collection := ""
|
|
if s3a.option.FilerGroup != "" {
|
|
collection = s3a.getCollectionName(bucket)
|
|
}
|
|
return func(reader io.Reader, name string, offset int64, tsNs int64, expectedDataSize uint64) (*filer_pb.FileChunk, error) {
|
|
return filer.SaveGatewayDataAsChunk(filer.GatewayChunkUploadRequest{
|
|
FilerClient: s3a,
|
|
Reader: reader,
|
|
FullPath: filePath,
|
|
Offset: offset,
|
|
TsNs: tsNs,
|
|
Collection: collection,
|
|
TtlSec: ttlSec,
|
|
DataCenter: s3a.option.DataCenter,
|
|
Cipher: s3a.cipher,
|
|
})
|
|
}
|
|
}
|
|
|
|
// manifestizeChunks folds a large flat chunk list into manifest chunks. A
|
|
// failed fold falls back to the flat list, deleting any blobs it saved.
|
|
func (s3a *S3ApiServer) manifestizeChunks(filePath string, bucket string, ttlSec int32, chunks []*filer_pb.FileChunk) []*filer_pb.FileChunk {
|
|
return manifestizeOrKeepFlat(s3a.saveManifestChunk(filePath, bucket, ttlSec), s3a.deleteOrphanedChunks, filePath, chunks)
|
|
}
|
|
|
|
func manifestizeOrKeepFlat(save filer.SaveDataAsChunkFunctionType, deleteChunks func([]*filer_pb.FileChunk), filePath string, chunks []*filer_pb.FileChunk) []*filer_pb.FileChunk {
|
|
var saved []*filer_pb.FileChunk
|
|
record := func(reader io.Reader, name string, offset int64, tsNs int64, expectedDataSize uint64) (*filer_pb.FileChunk, error) {
|
|
chunk, err := save(reader, name, offset, tsNs, expectedDataSize)
|
|
if err == nil {
|
|
saved = append(saved, chunk)
|
|
}
|
|
return chunk, err
|
|
}
|
|
manifested, err := filer.MaybeManifestize(record, chunks)
|
|
if err != nil {
|
|
glog.V(0).Infof("MaybeManifestize %s: %v", filePath, err)
|
|
if len(saved) > 0 {
|
|
deleteChunks(saved)
|
|
}
|
|
return chunks
|
|
}
|
|
return manifested
|
|
}
|
|
|
|
// flattenManifestChunks resolves manifest chunks into flat data chunks,
|
|
// returning the resolved-away manifests for callers that must delete the
|
|
// superseded blobs once they own the data chunks.
|
|
func (s3a *S3ApiServer) flattenManifestChunks(ctx context.Context, entry *filer_pb.Entry) ([]*filer_pb.FileChunk, error) {
|
|
if entry == nil || !filer.HasChunkManifest(entry.GetChunks()) {
|
|
return nil, nil
|
|
}
|
|
dataChunks, manifestChunks, err := filer.ResolveChunkManifest(ctx, s3a.createLookupFileIdFunction(), entry.GetChunks(), 0, math.MaxInt64)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
entry.Chunks = dataChunks
|
|
return manifestChunks, nil
|
|
}
|