From 6c1fd3aeab1df7c2d5fc323dda87ca284fffb596 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 6 Jun 2026 18:02:30 -0700 Subject: [PATCH] s3: rescan .versions when the cached latest pointer is missing on a list (#9841) * s3: rescan .versions when the cached latest pointer is missing on a list ListObjectsV2 resolves each versioned object's current version from the latest-version pointer cached on the .versions directory entry. When that pointer is absent on the filer serving the list, the object was dropped from the listing. Fall back to a read-only rescan of .versions/ to pick the newest version - the version files are present locally even when the cached pointer is not - so the object still lists. This mirrors the read path's recoverLatestVersionWithoutPointer; the scan loop is shared. Read-only by design: a list can touch many objects, so it does not persist a pointer. * s3: copy scanned Extended before stamping the version id --- weed/s3api/s3api_object_versioning.go | 119 +++++++++++++++++++------- 1 file changed, 88 insertions(+), 31 deletions(-) diff --git a/weed/s3api/s3api_object_versioning.go b/weed/s3api/s3api_object_versioning.go index ecbf12545..a1f0f4d83 100644 --- a/weed/s3api/s3api_object_versioning.go +++ b/weed/s3api/s3api_object_versioning.go @@ -1829,6 +1829,35 @@ func selectLatestVersion(entries []*filer_pb.Entry) (latestEntry *filer_pb.Entry return } +// scanLatestVersionEntry paginates a .versions/ directory and returns the +// chronologically newest version entry (including delete markers; see +// selectLatestVersion). A single-shot list would miss the true latest when +// old-format (raw timestamp) version ids spill past one page, since filesystem +// order is lexicographic-ascending = oldest-first for that format. latestEntry +// is nil when the directory holds no version entries. +func (s3a *S3ApiServer) scanLatestVersionEntry(versionsDir string) (latestEntry *filer_pb.Entry, latestVersionId, latestVersionFileName string, isDeleteMarker bool, err error) { + startFrom := "" + for { + entries, isLast, listErr := s3a.list(versionsDir, "", startFrom, false, filer.PaginationSize) + if listErr != nil { + return nil, "", "", false, fmt.Errorf("list %s: %w", versionsDir, listErr) + } + if pageEntry, pageId, pageFile, pageDM := selectLatestVersion(entries); pageEntry != nil { + if latestEntry == nil || compareVersionIds(pageId, latestVersionId) < 0 { + latestEntry = pageEntry + latestVersionId = pageId + latestVersionFileName = pageFile + isDeleteMarker = pageDM + } + } + if isLast || len(entries) == 0 { + break + } + startFrom = entries[len(entries)-1].Name + } + return +} + // healStaleLatestVersionPointer is invoked when the .versions directory metadata // points to a version file that no longer exists. It paginates the directory, // picks the chronologically newest remaining entry (content version or delete @@ -1854,30 +1883,9 @@ func (s3a *S3ApiServer) healStaleLatestVersionPointer(bucket, normalizedObject s // and the caller renders NoSuchKey (with x-amz-delete-marker) from the // returned entry. Restricting to content versions here would "undelete" // the object by promoting an older content version over a newer marker. - var ( - latestEntry *filer_pb.Entry - latestVersionId string - latestVersionFileName string - isDeleteMarker bool - startFrom string - ) - for { - entries, isLast, err := s3a.list(versionsDir, "", startFrom, false, filer.PaginationSize) - if err != nil { - return nil, fmt.Errorf("list %s: %w", versionsDir, err) - } - if pageEntry, pageId, pageFile, pageDM := selectLatestVersion(entries); pageEntry != nil { - if latestEntry == nil || compareVersionIds(pageId, latestVersionId) < 0 { - latestEntry = pageEntry - latestVersionId = pageId - latestVersionFileName = pageFile - isDeleteMarker = pageDM - } - } - if isLast || len(entries) == 0 { - break - } - startFrom = entries[len(entries)-1].Name + latestEntry, latestVersionId, latestVersionFileName, isDeleteMarker, scanErr := s3a.scanLatestVersionEntry(versionsDir) + if scanErr != nil { + return nil, scanErr } if latestEntry == nil { @@ -1932,14 +1940,15 @@ func (s3a *S3ApiServer) getLatestVersionEntryFromDirectoryEntry(bucket, object s normalizedObject := s3_constants.NormalizeObjectKey(object) - // Check if the directory entry has latest version metadata - if versionsDirEntry.Extended == nil { - return nil, fmt.Errorf("no Extended metadata in .versions directory entry") - } - + // The latest-version pointer is normally cached on the .versions directory + // entry, giving a single-scan listing. In a multi-filer deployment the pointer + // is written on the key's owner filer and may be absent on the filer serving + // this list, so recover by rescanning .versions/ rather than dropping the + // object from the listing entirely (the version files themselves replicate + // here). Indexing a nil Extended map is safe and yields !ok. latestVersionIdBytes, hasLatestVersionId := versionsDirEntry.Extended[s3_constants.ExtLatestVersionIdKey] if !hasLatestVersionId { - return nil, fmt.Errorf("missing latest version ID metadata in .versions directory entry") + return s3a.recoverLatestListEntryByScan(bucket, normalizedObject) } // Check if this is a delete marker (should not be shown in regular list) @@ -1998,7 +2007,7 @@ func (s3a *S3ApiServer) getLatestVersionEntryFromDirectoryEntry(bucket, object s // Fallback: fetch version file if cached metadata not available (for older versions) latestVersionFileBytes, hasLatestVersionFile := versionsDirEntry.Extended[s3_constants.ExtLatestVersionFileNameKey] if !hasLatestVersionFile { - return nil, fmt.Errorf("missing latest version file name metadata in .versions directory entry") + return s3a.recoverLatestListEntryByScan(bucket, normalizedObject) } latestVersionFile := string(latestVersionFileBytes) @@ -2031,6 +2040,54 @@ func (s3a *S3ApiServer) getLatestVersionEntryFromDirectoryEntry(bucket, object s return logicalEntry, nil } +// recoverLatestListEntryByScan rebuilds an object's current-version list entry by +// rescanning .versions/ when the cached latest-version pointer is missing on the +// filer serving the list. This is the listing-path counterpart to the read path's +// recoverLatestVersionWithoutPointer, and the cure for a multi-filer undercount: +// the pointer is written on the key's owner filer and may not be present on the +// serving filer, but the version files themselves replicate, so a local rescan +// resolves the current version. It is read-only on purpose — a single list can +// touch many objects, so it does not persist a pointer (which would amplify into +// a write per diverged object); convergence is handled on the write/replication +// side. Returns ErrDeleteMarker when the current version is a delete marker +// (excluded from a regular listing) and filer_pb.ErrNotFound when nothing remains. +func (s3a *S3ApiServer) recoverLatestListEntryByScan(bucket, normalizedObject string) (*filer_pb.Entry, error) { + bucketDir := s3a.bucketDir(bucket) + versionsDir := bucketDir + "/" + normalizedObject + s3_constants.VersionsFolder + + latestEntry, latestVersionId, _, isDeleteMarker, err := s3a.scanLatestVersionEntry(versionsDir) + if err != nil { + return nil, err + } + if latestEntry == nil { + // No version files remain. A pre-versioning / suspended "null" object at the + // base path is the current version if one exists. + if regularEntry, regularErr := s3a.getEntry(bucketDir, normalizedObject); regularErr == nil { + return regularEntry, nil + } + return nil, fmt.Errorf("%w: no current version for %s/%s", filer_pb.ErrNotFound, bucket, normalizedObject) + } + if isDeleteMarker { + return nil, ErrDeleteMarker + } + + // Present the version file as a logical entry at the object's base path, + // matching the cached fast path's output shape. Copy Extended rather than + // share the scanned entry's map, since we stamp the version id onto it. + extended := make(map[string][]byte, len(latestEntry.Extended)+1) + for k, v := range latestEntry.Extended { + extended[k] = v + } + extended[s3_constants.ExtVersionIdKey] = []byte(latestVersionId) + return &filer_pb.Entry{ + Name: path.Base(normalizedObject), + IsDirectory: false, + Attributes: latestEntry.Attributes, + Extended: extended, + Chunks: latestEntry.Chunks, + }, nil +} + // getObjectOwnerFromVersion extracts object owner information from version metadata func (s3a *S3ApiServer) getObjectOwnerFromVersion(version *ObjectVersion, bucket, objectKey string) CanonicalUser { // First try to get owner from the version's OwnerID field (extracted during listing)