Merge pull request #9736 from Lyndon-Li/cbt-bitmap-implementation
Run the E2E test on kind / get-go-version (push) Failing after 1m4s
Run the E2E test on kind / build (push) Has been skipped
Run the E2E test on kind / setup-test-matrix (push) Successful in 3s
Run the E2E test on kind / run-e2e-test (push) Has been skipped
Main CI / get-go-version (push) Successful in 13s
Main CI / Build (push) Failing after 43s
Close stale issues and PRs / stale (push) Successful in 14s
Trivy Nightly Scan / Trivy nightly scan (velero, main) (push) Failing after 1m37s
Trivy Nightly Scan / Trivy nightly scan (velero-plugin-for-aws, main) (push) Failing after 1m23s
Trivy Nightly Scan / Trivy nightly scan (velero-plugin-for-gcp, main) (push) Failing after 1m22s
Trivy Nightly Scan / Trivy nightly scan (velero-plugin-for-microsoft-azure, main) (push) Failing after 1m23s

CBT bitmap implementation
This commit is contained in:
Xun Jiang/Bruce Jiang
2026-05-21 13:19:42 +08:00
committed by GitHub
12 changed files with 1362 additions and 47 deletions
+1
View File
@@ -0,0 +1 @@
Add CBT bitmap implementation for block data mover
+3
View File
@@ -9,6 +9,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
github.com/RoaringBitmap/roaring v1.9.4
github.com/aws/aws-sdk-go-v2 v1.24.1
github.com/aws/aws-sdk-go-v2/config v1.26.3
github.com/aws/aws-sdk-go-v2/credentials v1.16.14
@@ -92,6 +93,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect
github.com/aws/smithy-go v1.19.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.12.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
@@ -145,6 +147,7 @@ require (
github.com/moby/term v0.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mxk/go-vss v1.2.1 // indirect
github.com/natefinch/atomic v1.0.1 // indirect
+6
View File
@@ -113,6 +113,8 @@ github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb0
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ=
github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -168,6 +170,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/bombsimon/logrusr/v3 v3.0.0 h1:tcAoLfuAhKP9npBxWzSdpsvKPQt1XV02nSf2lZA82TQ=
@@ -568,6 +572,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+171
View File
@@ -0,0 +1,171 @@
// Code generated by mockery; DO NOT EDIT.
// github.com/vektra/mockery
// template: testify
package mocks
import (
"context"
mock "github.com/stretchr/testify/mock"
"github.com/vmware-tanzu/velero/pkg/cbtservice"
)
// NewService creates a new instance of Service. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewService(t interface {
mock.TestingT
Cleanup(func())
}) *Service {
mock := &Service{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
// Service is an autogenerated mock type for the Service type
type Service struct {
mock.Mock
}
type Service_Expecter struct {
mock *mock.Mock
}
func (_m *Service) EXPECT() *Service_Expecter {
return &Service_Expecter{mock: &_m.Mock}
}
// GetAllocatedBlocks provides a mock function for the type Service
func (_mock *Service) GetAllocatedBlocks(ctx context.Context, snapshot string, record func([]cbtservice.Range) error) error {
ret := _mock.Called(ctx, snapshot, record)
if len(ret) == 0 {
panic("no return value specified for GetAllocatedBlocks")
}
var r0 error
if returnFunc, ok := ret.Get(0).(func(context.Context, string, func([]cbtservice.Range) error) error); ok {
r0 = returnFunc(ctx, snapshot, record)
} else {
r0 = ret.Error(0)
}
return r0
}
// Service_GetAllocatedBlocks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetAllocatedBlocks'
type Service_GetAllocatedBlocks_Call struct {
*mock.Call
}
// GetAllocatedBlocks is a helper method to define mock.On call
// - ctx context.Context
// - snapshot string
// - record func([]cbtservice.Range) error
func (_e *Service_Expecter) GetAllocatedBlocks(ctx interface{}, snapshot interface{}, record interface{}) *Service_GetAllocatedBlocks_Call {
return &Service_GetAllocatedBlocks_Call{Call: _e.mock.On("GetAllocatedBlocks", ctx, snapshot, record)}
}
func (_c *Service_GetAllocatedBlocks_Call) Run(run func(ctx context.Context, snapshot string, record func([]cbtservice.Range) error)) *Service_GetAllocatedBlocks_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 string
if args[1] != nil {
arg1 = args[1].(string)
}
var arg2 func([]cbtservice.Range) error
if args[2] != nil {
arg2 = args[2].(func([]cbtservice.Range) error)
}
run(
arg0,
arg1,
arg2,
)
})
return _c
}
func (_c *Service_GetAllocatedBlocks_Call) Return(err error) *Service_GetAllocatedBlocks_Call {
_c.Call.Return(err)
return _c
}
func (_c *Service_GetAllocatedBlocks_Call) RunAndReturn(run func(ctx context.Context, snapshot string, record func([]cbtservice.Range) error) error) *Service_GetAllocatedBlocks_Call {
_c.Call.Return(run)
return _c
}
// GetChangedBlocks provides a mock function for the type Service
func (_mock *Service) GetChangedBlocks(ctx context.Context, snapshot string, changeID string, record func([]cbtservice.Range) error) error {
ret := _mock.Called(ctx, snapshot, changeID, record)
if len(ret) == 0 {
panic("no return value specified for GetChangedBlocks")
}
var r0 error
if returnFunc, ok := ret.Get(0).(func(context.Context, string, string, func([]cbtservice.Range) error) error); ok {
r0 = returnFunc(ctx, snapshot, changeID, record)
} else {
r0 = ret.Error(0)
}
return r0
}
// Service_GetChangedBlocks_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetChangedBlocks'
type Service_GetChangedBlocks_Call struct {
*mock.Call
}
// GetChangedBlocks is a helper method to define mock.On call
// - ctx context.Context
// - snapshot string
// - changeID string
// - record func([]cbtservice.Range) error
func (_e *Service_Expecter) GetChangedBlocks(ctx interface{}, snapshot interface{}, changeID interface{}, record interface{}) *Service_GetChangedBlocks_Call {
return &Service_GetChangedBlocks_Call{Call: _e.mock.On("GetChangedBlocks", ctx, snapshot, changeID, record)}
}
func (_c *Service_GetChangedBlocks_Call) Run(run func(ctx context.Context, snapshot string, changeID string, record func([]cbtservice.Range) error)) *Service_GetChangedBlocks_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 context.Context
if args[0] != nil {
arg0 = args[0].(context.Context)
}
var arg1 string
if args[1] != nil {
arg1 = args[1].(string)
}
var arg2 string
if args[2] != nil {
arg2 = args[2].(string)
}
var arg3 func([]cbtservice.Range) error
if args[3] != nil {
arg3 = args[3].(func([]cbtservice.Range) error)
}
run(
arg0,
arg1,
arg2,
arg3,
)
})
return _c
}
func (_c *Service_GetChangedBlocks_Call) Return(err error) *Service_GetChangedBlocks_Call {
_c.Call.Return(err)
return _c
}
func (_c *Service_GetChangedBlocks_Call) RunAndReturn(run func(ctx context.Context, snapshot string, changeID string, record func([]cbtservice.Range) error) error) *Service_GetChangedBlocks_Call {
_c.Call.Return(run)
return _c
}
+2 -2
View File
@@ -20,8 +20,8 @@ import "context"
// Range defines the range of a change
type Range struct {
Offset int64
Length int64
Offset uint64
Length uint64
}
// SourceInfo is the information provided to the uploader, the uploader calls CBT service with this information
+88 -26
View File
@@ -16,40 +16,102 @@ limitations under the License.
package cbt
import "github.com/vmware-tanzu/velero/pkg/cbtservice"
import (
"math/bits"
// Bitmap defines the methods to store and iterate the CBT bitmap
type Bitmap interface {
// Set sets bits within the provided range
Set(cbtservice.Range)
"github.com/RoaringBitmap/roaring"
// SetFull sets all bits to the bitmap
SetFull()
"github.com/vmware-tanzu/velero/pkg/uploader/cbt/types"
)
// Snapshot returns snapshot of the bitmap
SourceID() string
const (
InvalidOffset64 = ^uint64(0)
)
// ChangeID returns the changeID of the bitmap
ChangeID() string
// Iterator returns the iterator for the CBT Bitmap
Iterator() Iterator
type bitmapImpl struct {
bitmap *roaring.Bitmap
blockSize uint
blockSizeLog int
length uint64
snapshot string
changeID string
volumeID string
}
// Iterator defines the methods to iterate the CBT bitmap and query the associated information
type Iterator interface {
// ChangeID returns the changeID of the bitmap
ChangeID() string
type bitmapIterator struct {
bitmapImpl
iterator roaring.IntPeekable
}
// Snapshot returns snapshot of the bitmap
Snapshot() string
func NewBitmap(blockSize uint, length uint64, snapshot string, changeID string, volumeID string) types.Bitmap {
return &bitmapImpl{
bitmap: roaring.New(),
blockSize: blockSize,
blockSizeLog: bits.Len(blockSize) - 1,
length: length,
snapshot: snapshot,
changeID: changeID,
volumeID: volumeID,
}
}
// BlockSize returns the granularity of the bitmap
BlockSize() int
func (c *bitmapImpl) Set(offset, length uint64) {
if offset >= c.length {
return
}
// Count returns the toal number of count in the bitmap
Count() uint64
if offset+length > c.length {
length = c.length - offset
}
// Next returns the offset of the next set block and whether it comes to the end of the iteration
Next() (int64, bool)
start := offset >> c.blockSizeLog
end := (offset + length + uint64(c.blockSize) - 1) >> c.blockSizeLog
c.bitmap.AddRange(start, end)
}
func (c *bitmapImpl) SetFull() {
start := uint64(0)
end := (c.length + uint64(c.blockSize) - 1) >> c.blockSizeLog
c.bitmap.AddRange(start, end)
}
func (c *bitmapImpl) Snapshot() string {
return c.snapshot
}
func (c *bitmapImpl) ChangeID() string {
return c.changeID
}
func (c *bitmapImpl) VolumeID() string {
return c.volumeID
}
func (c *bitmapImpl) Iterator() types.Iterator {
if c.bitmap == nil {
return nil
}
return &bitmapIterator{
bitmapImpl: *c,
iterator: c.bitmap.Iterator(),
}
}
func (c *bitmapIterator) Next() (uint64, bool) {
if !c.iterator.HasNext() {
return InvalidOffset64, false
}
return uint64(c.iterator.Next()) << c.blockSizeLog, true
}
func (c *bitmapIterator) Count() uint64 {
return c.bitmap.GetCardinality()
}
func (c *bitmapIterator) BlockSize() uint {
return c.blockSize
}
+256
View File
@@ -0,0 +1,256 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cbt
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBitmapProperties(t *testing.T) {
b := NewBitmap(1024*1024, 10000*1024*1024, "snap-1", "change-1", "vol-1")
assert.Equal(t, "snap-1", b.Snapshot())
assert.Equal(t, "change-1", b.ChangeID())
assert.Equal(t, "vol-1", b.VolumeID())
}
func TestBitmapSet(t *testing.T) {
const mb = 1024 * 1024
const gb = 1024 * 1024 * 1024
tests := []struct {
name string
blockSize uint
totalLength uint64
setCalls []struct{ offset, length uint64 }
expectedCount uint64
expectedNext []uint64
}{
{
name: "set single block within bounds",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{0, 1000},
},
expectedCount: 1,
expectedNext: []uint64{0},
},
{
name: "set exactly one block",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{0, mb},
},
expectedCount: 1,
expectedNext: []uint64{0},
},
{
name: "set overlapping two blocks",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{mb - 1, 2},
},
expectedCount: 2,
expectedNext: []uint64{0, mb},
},
{
name: "set multiple non-contiguous blocks",
blockSize: mb,
totalLength: 20 * gb,
setCalls: []struct{ offset, length uint64 }{
{0, 100},
{2 * mb, 100},
},
expectedCount: 2,
expectedNext: []uint64{0, 2 * mb},
},
{
name: "set completely out of bounds (offset >= length)",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{10 * gb, 100},
{15 * gb, 100},
},
expectedCount: 0,
expectedNext: []uint64{},
},
{
name: "set partially out of bounds (truncated)",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{10*gb - mb/2, mb}, // Starts in the last block, length pushes it out of bounds
},
expectedCount: 1, // Only the last block should be set
expectedNext: []uint64{10*gb - mb},
},
{
name: "set spanning entire length",
blockSize: mb,
totalLength: 3 * mb, // 3 blocks: 0-1MB, 1MB-2MB, 2MB-3MB
setCalls: []struct{ offset, length uint64 }{
{0, 3 * mb},
},
expectedCount: 3,
expectedNext: []uint64{0, mb, 2 * mb},
},
{
name: "set large contiguous range",
blockSize: mb,
totalLength: 100 * gb,
setCalls: []struct{ offset, length uint64 }{
{10 * mb, 5 * mb}, // Starts at 10MB, spans 5 full blocks
},
expectedCount: 5,
expectedNext: []uint64{10 * mb, 11 * mb, 12 * mb, 13 * mb, 14 * mb},
},
{
name: "set empty length",
blockSize: mb,
totalLength: 10 * gb,
setCalls: []struct{ offset, length uint64 }{
{mb, 0},
},
expectedCount: 0,
expectedNext: []uint64{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := NewBitmap(tt.blockSize, tt.totalLength, "snap-1", "change-1", "vol-1")
for _, call := range tt.setCalls {
b.Set(call.offset, call.length)
}
iter := b.Iterator()
require.NotNil(t, iter)
assert.Equal(t, tt.expectedCount, iter.Count())
var actualNext []uint64
for {
offset, hasNext := iter.Next()
if !hasNext {
assert.Equal(t, InvalidOffset64, offset)
break
}
actualNext = append(actualNext, offset)
}
if len(tt.expectedNext) == 0 {
assert.Empty(t, actualNext)
} else {
assert.Equal(t, tt.expectedNext, actualNext)
}
})
}
}
func TestBitmapSetFull(t *testing.T) {
const mb = 1024 * 1024
// Total length 3MB, blockSize 1MB. This means 3 blocks total:
// block 0: 0 - 1MB
// block 1: 1MB - 2MB
// block 2: 2MB - 3MB
b := NewBitmap(mb, 3*mb, "snap-1", "change-1", "vol-1")
b.SetFull()
iter := b.Iterator()
require.NotNil(t, iter)
assert.Equal(t, uint64(3), iter.Count())
expectedOffsets := []uint64{0, mb, 2 * mb}
var actualOffsets []uint64
for {
offset, hasNext := iter.Next()
if !hasNext {
break
}
actualOffsets = append(actualOffsets, offset)
}
assert.Equal(t, expectedOffsets, actualOffsets)
}
func TestBitmapIterator(t *testing.T) {
const mb = 1024 * 1024
const gb = 1024 * 1024 * 1024
b := NewBitmap(mb, 10*gb, "snap-1", "change-1", "vol-1")
// Set multiple ranges to test contiguous iteration
b.Set(mb, 100) // Block 1
b.Set(3*mb, 5*mb) // Blocks 3, 4, 5, 6, 7
b.Set(10*gb-mb, mb) // Last block
iter := b.Iterator()
require.NotNil(t, iter)
// Test iterator properties
assert.Equal(t, "snap-1", iter.Snapshot())
assert.Equal(t, "change-1", iter.ChangeID())
assert.Equal(t, "vol-1", iter.VolumeID())
assert.Equal(t, uint(mb), iter.BlockSize())
assert.Equal(t, uint64(7), iter.Count()) // 1 + 5 + 1 = 7 blocks
expectedOffsets := []uint64{
mb,
3 * mb, 4 * mb, 5 * mb, 6 * mb, 7 * mb,
10*gb - mb,
}
// Test iteration
var actualOffsets []uint64
for {
offset, hasNext := iter.Next()
if !hasNext {
assert.Equal(t, InvalidOffset64, offset)
break
}
actualOffsets = append(actualOffsets, offset)
}
assert.Equal(t, expectedOffsets, actualOffsets)
// Test end of iteration multiple times to ensure it stays exhausted
offset, hasNext := iter.Next()
assert.False(t, hasNext)
assert.Equal(t, InvalidOffset64, offset)
offset, hasNext = iter.Next()
assert.False(t, hasNext)
assert.Equal(t, InvalidOffset64, offset)
}
func TestBitmapIteratorNilBitmap(t *testing.T) {
// Directly create bitmapImpl with a nil roaring.Bitmap to test safety
b := &bitmapImpl{
bitmap: nil,
}
iter := b.Iterator()
assert.Nil(t, iter)
}
+31 -19
View File
@@ -19,31 +19,43 @@ package cbt
import (
"context"
"github.com/pkg/errors"
"github.com/vmware-tanzu/velero/pkg/cbtservice"
"github.com/vmware-tanzu/velero/pkg/uploader/cbt/types"
)
// SetBitmapOrFull translates the allocated/changed blocks from CBT service to the given bitmap or set the bitmap to full when error happens
func SetBitmapOrFull(ctx context.Context, service cbtservice.Service, bitmap Bitmap) error {
var err error
func SetBitmapOrFull(ctx context.Context, service cbtservice.Service, bitmap types.Bitmap) (err error) {
defer func() {
if err != nil {
bitmap.SetFull()
}
}()
if service == nil {
return errors.New("CBT service is absent")
}
if bitmap.Snapshot() == "" {
return errors.New("invalid snapshot")
}
if bitmap.ChangeID() == "" {
err = setFromAllocatedBlocks(ctx, service, bitmap)
} else {
err = setFromChangedBlocks(ctx, service, bitmap)
return errors.Wrapf(service.GetAllocatedBlocks(ctx, bitmap.Snapshot(), func(blocks []cbtservice.Range) error {
for _, b := range blocks {
bitmap.Set(b.Offset, b.Length)
}
return nil
}), "error getting allocated blocks from CBT service")
}
if err != nil {
bitmap.SetFull()
}
return errors.Wrapf(service.GetChangedBlocks(ctx, bitmap.Snapshot(), bitmap.ChangeID(), func(blocks []cbtservice.Range) error {
for _, b := range blocks {
bitmap.Set(b.Offset, b.Length)
}
return err
}
// TODO implement in following PRs
func setFromAllocatedBlocks(_ context.Context, _ cbtservice.Service, _ Bitmap) error {
return nil
}
// TODO implement in following PRs
func setFromChangedBlocks(_ context.Context, _ cbtservice.Service, _ Bitmap) error {
return nil
return nil
}), "error getting changed blocks from CBT service")
}
+142
View File
@@ -0,0 +1,142 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cbt
import (
"context"
"errors"
"testing"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/vmware-tanzu/velero/pkg/cbtservice"
cbtservicemocks "github.com/vmware-tanzu/velero/pkg/cbtservice/mocks"
cbtmocks "github.com/vmware-tanzu/velero/pkg/uploader/cbt/types/mocks"
)
func TestSetBitmapOrFull(t *testing.T) {
tests := []struct {
name string
nilService bool
setupMocks func(*cbtservicemocks.Service, *cbtmocks.Bitmap)
expectedErrStr string
}{
{
name: "nil service",
nilService: true,
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("SetFull").Return()
},
expectedErrStr: "CBT service is absent",
},
{
name: "invalid snapshot",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("")
bmp.On("SetFull").Return()
},
expectedErrStr: "invalid snapshot",
},
{
name: "allocated blocks success",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("")
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Run(func(args mock.Arguments) {
record := args.Get(2).(func([]cbtservice.Range) error)
record([]cbtservice.Range{
{Offset: 0, Length: 4096},
{Offset: 8192, Length: 4096},
})
}).Return(nil)
bmp.On("Set", uint64(0), uint64(4096)).Return()
bmp.On("Set", uint64(8192), uint64(4096)).Return()
},
},
{
name: "allocated blocks error",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("")
svc.On("GetAllocatedBlocks", mock.Anything, "snap-1", mock.Anything).Return(errors.New("mock alloc error"))
bmp.On("SetFull").Return()
},
expectedErrStr: "error getting allocated blocks from CBT service: mock alloc error",
},
{
name: "changed blocks success",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("change-1")
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Run(func(args mock.Arguments) {
record := args.Get(3).(func([]cbtservice.Range) error)
record([]cbtservice.Range{
{Offset: 4096, Length: 4096},
})
}).Return(nil)
bmp.On("Set", uint64(4096), uint64(4096)).Return()
},
},
{
name: "changed blocks error",
setupMocks: func(svc *cbtservicemocks.Service, bmp *cbtmocks.Bitmap) {
bmp.On("Snapshot").Return("snap-1")
bmp.On("ChangeID").Return("change-1")
svc.On("GetChangedBlocks", mock.Anything, "snap-1", "change-1", mock.Anything).Return(errors.New("mock changed error"))
bmp.On("SetFull").Return()
},
expectedErrStr: "error getting changed blocks from CBT service: mock changed error",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
svcMock := new(cbtservicemocks.Service)
bmpMock := new(cbtmocks.Bitmap)
if tt.setupMocks != nil {
tt.setupMocks(svcMock, bmpMock)
}
var svc cbtservice.Service
if !tt.nilService {
svc = svcMock
}
err := SetBitmapOrFull(context.Background(), svc, bmpMock)
if tt.expectedErrStr != "" {
require.Error(t, err)
require.EqualError(t, err, tt.expectedErrStr)
} else {
require.NoError(t, err)
}
if !tt.nilService {
svcMock.AssertExpectations(t)
}
bmpMock.AssertExpectations(t)
})
}
}
+294
View File
@@ -0,0 +1,294 @@
// Code generated by mockery; DO NOT EDIT.
// github.com/vektra/mockery
// template: testify
package mocks
import (
mock "github.com/stretchr/testify/mock"
"github.com/vmware-tanzu/velero/pkg/uploader/cbt/types"
)
// NewBitmap creates a new instance of Bitmap. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewBitmap(t interface {
mock.TestingT
Cleanup(func())
}) *Bitmap {
mock := &Bitmap{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
// Bitmap is an autogenerated mock type for the Bitmap type
type Bitmap struct {
mock.Mock
}
type Bitmap_Expecter struct {
mock *mock.Mock
}
func (_m *Bitmap) EXPECT() *Bitmap_Expecter {
return &Bitmap_Expecter{mock: &_m.Mock}
}
// ChangeID provides a mock function for the type Bitmap
func (_mock *Bitmap) ChangeID() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for ChangeID")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Bitmap_ChangeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChangeID'
type Bitmap_ChangeID_Call struct {
*mock.Call
}
// ChangeID is a helper method to define mock.On call
func (_e *Bitmap_Expecter) ChangeID() *Bitmap_ChangeID_Call {
return &Bitmap_ChangeID_Call{Call: _e.mock.On("ChangeID")}
}
func (_c *Bitmap_ChangeID_Call) Run(run func()) *Bitmap_ChangeID_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Bitmap_ChangeID_Call) Return(s string) *Bitmap_ChangeID_Call {
_c.Call.Return(s)
return _c
}
func (_c *Bitmap_ChangeID_Call) RunAndReturn(run func() string) *Bitmap_ChangeID_Call {
_c.Call.Return(run)
return _c
}
// Iterator provides a mock function for the type Bitmap
func (_mock *Bitmap) Iterator() types.Iterator {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Iterator")
}
var r0 types.Iterator
if returnFunc, ok := ret.Get(0).(func() types.Iterator); ok {
r0 = returnFunc()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(types.Iterator)
}
}
return r0
}
// Bitmap_Iterator_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Iterator'
type Bitmap_Iterator_Call struct {
*mock.Call
}
// Iterator is a helper method to define mock.On call
func (_e *Bitmap_Expecter) Iterator() *Bitmap_Iterator_Call {
return &Bitmap_Iterator_Call{Call: _e.mock.On("Iterator")}
}
func (_c *Bitmap_Iterator_Call) Run(run func()) *Bitmap_Iterator_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Bitmap_Iterator_Call) Return(iterator types.Iterator) *Bitmap_Iterator_Call {
_c.Call.Return(iterator)
return _c
}
func (_c *Bitmap_Iterator_Call) RunAndReturn(run func() types.Iterator) *Bitmap_Iterator_Call {
_c.Call.Return(run)
return _c
}
// Set provides a mock function for the type Bitmap
func (_mock *Bitmap) Set(v uint64, v1 uint64) {
_mock.Called(v, v1)
return
}
// Bitmap_Set_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Set'
type Bitmap_Set_Call struct {
*mock.Call
}
// Set is a helper method to define mock.On call
// - v uint64
// - v1 uint64
func (_e *Bitmap_Expecter) Set(v interface{}, v1 interface{}) *Bitmap_Set_Call {
return &Bitmap_Set_Call{Call: _e.mock.On("Set", v, v1)}
}
func (_c *Bitmap_Set_Call) Run(run func(v uint64, v1 uint64)) *Bitmap_Set_Call {
_c.Call.Run(func(args mock.Arguments) {
var arg0 uint64
if args[0] != nil {
arg0 = args[0].(uint64)
}
var arg1 uint64
if args[1] != nil {
arg1 = args[1].(uint64)
}
run(
arg0,
arg1,
)
})
return _c
}
func (_c *Bitmap_Set_Call) Return() *Bitmap_Set_Call {
_c.Call.Return()
return _c
}
func (_c *Bitmap_Set_Call) RunAndReturn(run func(v uint64, v1 uint64)) *Bitmap_Set_Call {
_c.Run(run)
return _c
}
// SetFull provides a mock function for the type Bitmap
func (_mock *Bitmap) SetFull() {
_mock.Called()
return
}
// Bitmap_SetFull_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SetFull'
type Bitmap_SetFull_Call struct {
*mock.Call
}
// SetFull is a helper method to define mock.On call
func (_e *Bitmap_Expecter) SetFull() *Bitmap_SetFull_Call {
return &Bitmap_SetFull_Call{Call: _e.mock.On("SetFull")}
}
func (_c *Bitmap_SetFull_Call) Run(run func()) *Bitmap_SetFull_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Bitmap_SetFull_Call) Return() *Bitmap_SetFull_Call {
_c.Call.Return()
return _c
}
func (_c *Bitmap_SetFull_Call) RunAndReturn(run func()) *Bitmap_SetFull_Call {
_c.Run(run)
return _c
}
// Snapshot provides a mock function for the type Bitmap
func (_mock *Bitmap) Snapshot() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Snapshot")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Bitmap_Snapshot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Snapshot'
type Bitmap_Snapshot_Call struct {
*mock.Call
}
// Snapshot is a helper method to define mock.On call
func (_e *Bitmap_Expecter) Snapshot() *Bitmap_Snapshot_Call {
return &Bitmap_Snapshot_Call{Call: _e.mock.On("Snapshot")}
}
func (_c *Bitmap_Snapshot_Call) Run(run func()) *Bitmap_Snapshot_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Bitmap_Snapshot_Call) Return(s string) *Bitmap_Snapshot_Call {
_c.Call.Return(s)
return _c
}
func (_c *Bitmap_Snapshot_Call) RunAndReturn(run func() string) *Bitmap_Snapshot_Call {
_c.Call.Return(run)
return _c
}
// VolumeID provides a mock function for the type Bitmap
func (_mock *Bitmap) VolumeID() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for VolumeID")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Bitmap_VolumeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'VolumeID'
type Bitmap_VolumeID_Call struct {
*mock.Call
}
// VolumeID is a helper method to define mock.On call
func (_e *Bitmap_Expecter) VolumeID() *Bitmap_VolumeID_Call {
return &Bitmap_VolumeID_Call{Call: _e.mock.On("VolumeID")}
}
func (_c *Bitmap_VolumeID_Call) Run(run func()) *Bitmap_VolumeID_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Bitmap_VolumeID_Call) Return(s string) *Bitmap_VolumeID_Call {
_c.Call.Return(s)
return _c
}
func (_c *Bitmap_VolumeID_Call) RunAndReturn(run func() string) *Bitmap_VolumeID_Call {
_c.Call.Return(run)
return _c
}
+309
View File
@@ -0,0 +1,309 @@
// Code generated by mockery; DO NOT EDIT.
// github.com/vektra/mockery
// template: testify
package mocks
import (
mock "github.com/stretchr/testify/mock"
)
// NewIterator creates a new instance of Iterator. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewIterator(t interface {
mock.TestingT
Cleanup(func())
}) *Iterator {
mock := &Iterator{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}
// Iterator is an autogenerated mock type for the Iterator type
type Iterator struct {
mock.Mock
}
type Iterator_Expecter struct {
mock *mock.Mock
}
func (_m *Iterator) EXPECT() *Iterator_Expecter {
return &Iterator_Expecter{mock: &_m.Mock}
}
// BlockSize provides a mock function for the type Iterator
func (_mock *Iterator) BlockSize() uint {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for BlockSize")
}
var r0 uint
if returnFunc, ok := ret.Get(0).(func() uint); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(uint)
}
return r0
}
// Iterator_BlockSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BlockSize'
type Iterator_BlockSize_Call struct {
*mock.Call
}
// BlockSize is a helper method to define mock.On call
func (_e *Iterator_Expecter) BlockSize() *Iterator_BlockSize_Call {
return &Iterator_BlockSize_Call{Call: _e.mock.On("BlockSize")}
}
func (_c *Iterator_BlockSize_Call) Run(run func()) *Iterator_BlockSize_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_BlockSize_Call) Return(v uint) *Iterator_BlockSize_Call {
_c.Call.Return(v)
return _c
}
func (_c *Iterator_BlockSize_Call) RunAndReturn(run func() uint) *Iterator_BlockSize_Call {
_c.Call.Return(run)
return _c
}
// ChangeID provides a mock function for the type Iterator
func (_mock *Iterator) ChangeID() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for ChangeID")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Iterator_ChangeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChangeID'
type Iterator_ChangeID_Call struct {
*mock.Call
}
// ChangeID is a helper method to define mock.On call
func (_e *Iterator_Expecter) ChangeID() *Iterator_ChangeID_Call {
return &Iterator_ChangeID_Call{Call: _e.mock.On("ChangeID")}
}
func (_c *Iterator_ChangeID_Call) Run(run func()) *Iterator_ChangeID_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_ChangeID_Call) Return(s string) *Iterator_ChangeID_Call {
_c.Call.Return(s)
return _c
}
func (_c *Iterator_ChangeID_Call) RunAndReturn(run func() string) *Iterator_ChangeID_Call {
_c.Call.Return(run)
return _c
}
// Count provides a mock function for the type Iterator
func (_mock *Iterator) Count() uint64 {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Count")
}
var r0 uint64
if returnFunc, ok := ret.Get(0).(func() uint64); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(uint64)
}
return r0
}
// Iterator_Count_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Count'
type Iterator_Count_Call struct {
*mock.Call
}
// Count is a helper method to define mock.On call
func (_e *Iterator_Expecter) Count() *Iterator_Count_Call {
return &Iterator_Count_Call{Call: _e.mock.On("Count")}
}
func (_c *Iterator_Count_Call) Run(run func()) *Iterator_Count_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_Count_Call) Return(v uint64) *Iterator_Count_Call {
_c.Call.Return(v)
return _c
}
func (_c *Iterator_Count_Call) RunAndReturn(run func() uint64) *Iterator_Count_Call {
_c.Call.Return(run)
return _c
}
// Next provides a mock function for the type Iterator
func (_mock *Iterator) Next() (uint64, bool) {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Next")
}
var r0 uint64
var r1 bool
if returnFunc, ok := ret.Get(0).(func() (uint64, bool)); ok {
return returnFunc()
}
if returnFunc, ok := ret.Get(0).(func() uint64); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(uint64)
}
if returnFunc, ok := ret.Get(1).(func() bool); ok {
r1 = returnFunc()
} else {
r1 = ret.Get(1).(bool)
}
return r0, r1
}
// Iterator_Next_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Next'
type Iterator_Next_Call struct {
*mock.Call
}
// Next is a helper method to define mock.On call
func (_e *Iterator_Expecter) Next() *Iterator_Next_Call {
return &Iterator_Next_Call{Call: _e.mock.On("Next")}
}
func (_c *Iterator_Next_Call) Run(run func()) *Iterator_Next_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_Next_Call) Return(v uint64, b bool) *Iterator_Next_Call {
_c.Call.Return(v, b)
return _c
}
func (_c *Iterator_Next_Call) RunAndReturn(run func() (uint64, bool)) *Iterator_Next_Call {
_c.Call.Return(run)
return _c
}
// Snapshot provides a mock function for the type Iterator
func (_mock *Iterator) Snapshot() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for Snapshot")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Iterator_Snapshot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Snapshot'
type Iterator_Snapshot_Call struct {
*mock.Call
}
// Snapshot is a helper method to define mock.On call
func (_e *Iterator_Expecter) Snapshot() *Iterator_Snapshot_Call {
return &Iterator_Snapshot_Call{Call: _e.mock.On("Snapshot")}
}
func (_c *Iterator_Snapshot_Call) Run(run func()) *Iterator_Snapshot_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_Snapshot_Call) Return(s string) *Iterator_Snapshot_Call {
_c.Call.Return(s)
return _c
}
func (_c *Iterator_Snapshot_Call) RunAndReturn(run func() string) *Iterator_Snapshot_Call {
_c.Call.Return(run)
return _c
}
// VolumeID provides a mock function for the type Iterator
func (_mock *Iterator) VolumeID() string {
ret := _mock.Called()
if len(ret) == 0 {
panic("no return value specified for VolumeID")
}
var r0 string
if returnFunc, ok := ret.Get(0).(func() string); ok {
r0 = returnFunc()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Iterator_VolumeID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'VolumeID'
type Iterator_VolumeID_Call struct {
*mock.Call
}
// VolumeID is a helper method to define mock.On call
func (_e *Iterator_Expecter) VolumeID() *Iterator_VolumeID_Call {
return &Iterator_VolumeID_Call{Call: _e.mock.On("VolumeID")}
}
func (_c *Iterator_VolumeID_Call) Run(run func()) *Iterator_VolumeID_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *Iterator_VolumeID_Call) Return(s string) *Iterator_VolumeID_Call {
_c.Call.Return(s)
return _c
}
func (_c *Iterator_VolumeID_Call) RunAndReturn(run func() string) *Iterator_VolumeID_Call {
_c.Call.Return(run)
return _c
}
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package types
// Bitmap defines the methods to store and iterate the CBT bitmap
type Bitmap interface {
// Set sets bits within the provided range
Set(uint64, uint64)
// SetFull sets all bits to the bitmap
SetFull()
// Snapshot returns snapshot of the bitmap
Snapshot() string
// ChangeID returns the changeID of the bitmap
ChangeID() string
// VolumeID return ID of the volume from which the snapshot is taken
VolumeID() string
// Iterator returns the iterator for the CBT Bitmap
Iterator() Iterator
}
// Iterator defines the methods to iterate the CBT bitmap and query the associated information
type Iterator interface {
// ChangeID returns the changeID of the bitmap
ChangeID() string
// Snapshot returns snapshot of the bitmap
Snapshot() string
// VolumeID return ID of the volume from which the snapshot is taken
VolumeID() string
// BlockSize returns the granularity of the bitmap
BlockSize() uint
// Count returns the total number of count in the bitmap
Count() uint64
// Next returns the offset of the next set block and whether it comes to the end of the iteration
Next() (uint64, bool)
}