diff --git a/cmd/metacache-entries.go b/cmd/metacache-entries.go index f0f26534b..c3eec6738 100644 --- a/cmd/metacache-entries.go +++ b/cmd/metacache-entries.go @@ -281,10 +281,16 @@ func (m metaCacheEntries) shallowClone() metaCacheEntries { } type metadataResolutionParams struct { - dirQuorum int // Number if disks needed for a directory to 'exist'. - objQuorum int // Number of disks needed for an object to 'exist'. - bucket string // Name of the bucket. Used for generating cached fileinfo. - strict bool // Versions must match exactly, including all metadata. + dirQuorum int // Number if disks needed for a directory to 'exist'. + objQuorum int // Number of disks needed for an object to 'exist'. + + // An optimization request only an 'n' amount of versions from xl.meta + // to avoid resolving all versions to figure out the latest 'version' + // for ListObjects, ListObjectsV2 + requestedVersions int + + bucket string // Name of the bucket. Used for generating cached fileinfo. + strict bool // Versions must match exactly, including all metadata. // Reusable slice for resolution candidates [][]xlMetaV2ShallowVersion @@ -372,7 +378,7 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa reusable: true, cached: &xlMetaV2{metaV: selected.cached.metaV}, } - selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.candidates...) + selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.requestedVersions, r.candidates...) if len(selected.cached.versions) == 0 { return nil, false } diff --git a/cmd/metacache-set.go b/cmd/metacache-set.go index 7a03c081a..f73bcfd28 100644 --- a/cmd/metacache-set.go +++ b/cmd/metacache-set.go @@ -571,6 +571,13 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul bucket: o.Bucket, } + // Maximum versions requested for "latest" object + // resolution on versioned buckets, this is to be only + // used when o.Versioned is false + if !o.Versioned { + resolver.requestedVersions = 1 + } + ctxDone := ctx.Done() return listPathRaw(ctx, listPathRawOptions{ disks: disks, diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 2e5ea3d0c..4d04cb841 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -1691,7 +1691,7 @@ func (x xlMetaV2) ListVersions(volume, path string) ([]FileInfo, error) { // Quorum must be the minimum number of matching metadata files. // Quorum should be > 1 and <= len(versions). // If strict is set to false, entries that match type -func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVersion) (merged []xlMetaV2ShallowVersion) { +func mergeXLV2Versions(quorum int, strict bool, requestedVersions int, versions ...[]xlMetaV2ShallowVersion) (merged []xlMetaV2ShallowVersion) { if quorum <= 0 { quorum = 1 } @@ -1708,6 +1708,8 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer // Shallow copy input versions = append(make([][]xlMetaV2ShallowVersion, 0, len(versions)), versions...) + var nVersions int // captures all non-free versions + // Our result merged = make([]xlMetaV2ShallowVersion, 0, len(versions[0])) tops := make([]xlMetaV2ShallowVersion, len(versions)) @@ -1744,6 +1746,12 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer latest = tops[0] latestCount = len(tops) merged = append(merged, latest) + + // Calculate latest 'n' non-free versions. + if !latest.header.FreeVersion() { + nVersions++ + } + } else { // Find latest. for i, ver := range tops { @@ -1807,6 +1815,11 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer } if latestCount >= quorum { merged = append(merged, latest) + + // Calculate latest 'n' non-free versions. + if !latest.header.FreeVersion() { + nVersions++ + } } } @@ -1840,7 +1853,13 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer break } } + + if requestedVersions > 0 && requestedVersions == nVersions { + merged = append(merged, versions[0]...) + break + } } + // Sanity check. Enable if duplicates show up. if false { found := make(map[[16]byte]struct{}) diff --git a/cmd/xl-storage-format-v2_test.go b/cmd/xl-storage-format-v2_test.go index c5727210c..5516dbe0f 100644 --- a/cmd/xl-storage-format-v2_test.go +++ b/cmd/xl-storage-format-v2_test.go @@ -401,6 +401,55 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) { } } +func Benchmark_mergeXLV2Versions(b *testing.B) { + data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst") + if err != nil { + b.Fatal(err) + } + dec, _ := zstd.NewReader(nil) + data, err = dec.DecodeAll(data, nil) + if err != nil { + b.Fatal(err) + } + + var xl xlMetaV2 + if err = xl.LoadOrConvert(data); err != nil { + b.Fatal(err) + } + + vers := make([][]xlMetaV2ShallowVersion, 16) + for i := range vers { + vers[i] = xl.versions + } + + b.Run("requested-none", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(855) // number of versions... + for i := 0; i < b.N; i++ { + mergeXLV2Versions(8, false, 0, vers...) + } + }) + + b.Run("requested-v1", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(855) // number of versions... + for i := 0; i < b.N; i++ { + mergeXLV2Versions(8, false, 1, vers...) + } + }) + + b.Run("requested-v2", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(855) // number of versions... + for i := 0; i < b.N; i++ { + mergeXLV2Versions(8, false, 1, vers...) + } + }) +} + func Benchmark_xlMetaV2Shallow_Load(b *testing.B) { data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst") if err != nil { @@ -424,6 +473,7 @@ func Benchmark_xlMetaV2Shallow_Load(b *testing.B) { } } }) + b.Run("indexed", func(b *testing.B) { var xl xlMetaV2 err = xl.Load(data) @@ -550,7 +600,7 @@ func Test_mergeXLV2Versions(t *testing.T) { for i := range vers { t.Run(fmt.Sprintf("non-strict-q%d", i), func(t *testing.T) { - merged := mergeXLV2Versions(i, false, vers...) + merged := mergeXLV2Versions(i, false, 0, vers...) if len(merged) == 0 { t.Error("Did not get any results") return @@ -562,7 +612,7 @@ func Test_mergeXLV2Versions(t *testing.T) { } }) t.Run(fmt.Sprintf("strict-q%d", i), func(t *testing.T) { - merged := mergeXLV2Versions(i, true, vers...) + merged := mergeXLV2Versions(i, true, 0, vers...) if len(merged) == 0 { t.Error("Did not get any results") return @@ -584,7 +634,7 @@ func Test_mergeXLV2Versions(t *testing.T) { } vMod = append(vMod, newVers) } - merged := mergeXLV2Versions(i, false, vMod...) + merged := mergeXLV2Versions(i, false, 0, vMod...) if len(merged) == 0 { t.Error("Did not get any results") return @@ -606,7 +656,7 @@ func Test_mergeXLV2Versions(t *testing.T) { } vMod = append(vMod, newVers) } - merged := mergeXLV2Versions(i, false, vMod...) + merged := mergeXLV2Versions(i, false, 0, vMod...) if len(merged) == 0 && i < 2 { t.Error("Did not get any results") return @@ -632,7 +682,7 @@ func Test_mergeXLV2Versions(t *testing.T) { } vMod = append(vMod, newVers) } - merged := mergeXLV2Versions(i, false, vMod...) + merged := mergeXLV2Versions(i, false, 0, vMod...) if len(merged) == 0 { t.Error("Did not get any results") return @@ -654,7 +704,7 @@ func Test_mergeXLV2Versions(t *testing.T) { } vMod = append(vMod, newVers) } - merged := mergeXLV2Versions(i, false, vMod...) + merged := mergeXLV2Versions(i, false, 0, vMod...) if len(merged) == 0 && i < 2 { t.Error("Did not get any results") return @@ -680,7 +730,7 @@ func Test_mergeXLV2Versions(t *testing.T) { } vMod = append(vMod, newVers) } - merged := mergeXLV2Versions(i, true, vMod...) + merged := mergeXLV2Versions(i, true, 0, vMod...) if len(merged) == 0 && i < 2 { t.Error("Did not get any results") return @@ -706,7 +756,7 @@ func Test_mergeXLV2Versions(t *testing.T) { } vMod = append(vMod, newVers) } - merged := mergeXLV2Versions(i, true, vMod...) + merged := mergeXLV2Versions(i, true, 0, vMod...) if len(merged) == 0 && i < 2 { t.Error("Did not get any results") return @@ -732,7 +782,7 @@ func Test_mergeXLV2Versions(t *testing.T) { } vMod = append(vMod, newVers) } - merged := mergeXLV2Versions(i, true, vMod...) + merged := mergeXLV2Versions(i, true, 0, vMod...) if len(merged) == 0 && i < 2 { t.Error("Did not get any results") return @@ -758,7 +808,7 @@ func Test_mergeXLV2Versions(t *testing.T) { } vMod = append(vMod, newVers) } - merged := mergeXLV2Versions(i, true, vMod...) + merged := mergeXLV2Versions(i, true, 0, vMod...) if len(merged) == 0 && i < 2 { t.Error("Did not get any results") return