diff --git a/changelogs/unreleased/9808-Lyndon-Li b/changelogs/unreleased/9808-Lyndon-Li new file mode 100644 index 000000000..d3564ab2d --- /dev/null +++ b/changelogs/unreleased/9808-Lyndon-Li @@ -0,0 +1 @@ +Add Kopia repo snapshot operations for block data mover \ No newline at end of file diff --git a/pkg/repository/udmrepo/kopialib/lib_repo.go b/pkg/repository/udmrepo/kopialib/lib_repo.go index cb16506f3..2131baa5c 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo.go @@ -509,19 +509,116 @@ func (kr *kopiaRepository) DeleteManifest(ctx context.Context, id udmrepo.ID) er return nil } -// TODO add implementation in following PRs func (kr *kopiaRepository) SaveSnapshot(ctx context.Context, snap udmrepo.Snapshot) (udmrepo.ID, error) { - return "", errors.New("not supported") + if kr.rawWriter == nil { + return "", errors.New("repo writer is closed or not open") + } + + if snap.Source == "" { + return "", errors.New("invalid snapshot source") + } + + rootObj, err := object.ParseID(string(snap.RootObject.ID)) + if err != nil { + return "", errors.Wrapf(err, "error parsing root object ID %v", snap.RootObject.ID) + } + + manifest := snapshot.Manifest{ + Source: snapshot.SourceInfo{ + UserName: udmrepo.GetRepoUser(), + Host: udmrepo.GetRepoDomain(), + Path: snap.Source, + }, + Description: snap.Description, + StartTime: fs.UTCTimestampFromTime(snap.StartTime), + EndTime: fs.UTCTimestampFromTime(snap.EndTime), + RootEntry: &snapshot.DirEntry{ + Type: snapshot.EntryTypeDirectory, + ObjectID: rootObj, + ModTime: fs.UTCTimestampFromTime(snap.RootObject.ModTime), + Permissions: snapshot.Permissions(snap.RootObject.Permissions), + FileSize: snap.RootObject.Size, + UserID: snap.RootObject.UserID, + GroupID: snap.RootObject.GroupID, + }, + Tags: snap.Tags, + } + + id, err := snapshot.SaveSnapshot(ctx, kr.rawWriter, &manifest) + if err != nil { + return "", errors.Wrap(err, "error saving snapshot") + } + + return udmrepo.ID(id), nil } -// TODO add implementation in following PRs func (kr *kopiaRepository) GetSnapshot(ctx context.Context, id udmrepo.ID) (udmrepo.Snapshot, error) { - return udmrepo.Snapshot{}, errors.New("not supported") + snap, err := snapshot.LoadSnapshot(ctx, kr.rawRepo, manifest.ID(id)) + if err != nil { + return udmrepo.Snapshot{}, errors.Wrap(err, "error getting snapshot manifest") + } + + if snap.RootEntry == nil { + return udmrepo.Snapshot{}, errors.New("invalid snapshot root entry") + } + + return udmrepo.Snapshot{ + Source: snap.Source.Path, + Description: snap.Description, + StartTime: snap.StartTime.ToTime(), + EndTime: snap.EndTime.ToTime(), + Tags: snap.Tags, + RootObject: udmrepo.ObjectMetadata{ + ID: udmrepo.ID(snap.RootEntry.ObjectID.String()), + Type: udmrepo.ObjectDataTypeMetadata, + Size: snap.RootEntry.FileSize, + ModTime: snap.RootEntry.ModTime.ToTime(), + Permissions: int(snap.RootEntry.Permissions), + UserID: snap.RootEntry.UserID, + GroupID: snap.RootEntry.GroupID, + }, + }, nil } -// TODO add implementation in following PRs func (kr *kopiaRepository) DeleteSnapshot(ctx context.Context, id udmrepo.ID) error { - return errors.New("not supported") + if _, err := kr.GetSnapshot(ctx, id); err != nil { + return errors.Wrap(err, "error getting snapshot") + } + + return kr.DeleteManifest(ctx, id) +} + +func (kr *kopiaRepository) ListSnapshot(ctx context.Context, source string) ([]udmrepo.Snapshot, error) { + mani, err := snapshot.ListSnapshots(ctx, kr.rawRepo, snapshot.SourceInfo{ + Host: udmrepo.GetRepoDomain(), + UserName: udmrepo.GetRepoUser(), + Path: source, + }) + if err != nil { + return nil, errors.Wrapf(err, "error listing snapshot manifest for source %s", source) + } + + snapshots := []udmrepo.Snapshot{} + for _, snap := range mani { + snapshots = append(snapshots, udmrepo.Snapshot{ + Source: snap.Source.Path, + Description: snap.Description, + StartTime: snap.StartTime.ToTime(), + EndTime: snap.EndTime.ToTime(), + Tags: snap.Tags, + RootObject: udmrepo.ObjectMetadata{ + ID: udmrepo.ID(snap.RootEntry.ObjectID.String()), + Type: udmrepo.ObjectDataTypeMetadata, + Size: snap.RootEntry.FileSize, + ModTime: snap.RootEntry.ModTime.ToTime(), + Permissions: int(snap.RootEntry.Permissions), + UserID: snap.RootEntry.UserID, + GroupID: snap.RootEntry.GroupID, + }, + }) + } + + return snapshots, nil } func (kr *kopiaRepository) Flush(ctx context.Context) error { diff --git a/pkg/repository/udmrepo/kopialib/lib_repo_test.go b/pkg/repository/udmrepo/kopialib/lib_repo_test.go index 1b8d5f376..1776f9be0 100644 --- a/pkg/repository/udmrepo/kopialib/lib_repo_test.go +++ b/pkg/repository/udmrepo/kopialib/lib_repo_test.go @@ -19,14 +19,17 @@ package kopialib import ( "bytes" "context" + "encoding/json" "math" "os" "testing" "time" + "github.com/kopia/kopia/fs" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/repo/object" + "github.com/kopia/kopia/snapshot" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -1450,3 +1453,382 @@ func TestReadMetadata(t *testing.T) { }) } } + +func TestSaveSnapshot(t *testing.T) { + testCases := []struct { + name string + rawWriter *repomocks.MockRepositoryWriter + snap udmrepo.Snapshot + rawWriterRetErr error + rawWriterRetID manifest.ID + setWriterMock bool + expectedErr string + expectedID udmrepo.ID + }{ + { + name: "raw writer is nil", + expectedErr: "repo writer is closed or not open", + }, + { + name: "invalid snapshot source", + rawWriter: repomocks.NewMockRepositoryWriter(t), + snap: udmrepo.Snapshot{ + Source: "", + }, + expectedErr: "invalid snapshot source", + }, + { + name: "invalid root object id", + rawWriter: repomocks.NewMockRepositoryWriter(t), + snap: udmrepo.Snapshot{ + Source: "fake-source", + RootObject: udmrepo.ObjectMetadata{ID: "fake-id"}, + }, + expectedErr: "error parsing root object ID fake-id: malformed content ID: \"fake-id\": invalid content prefix", + }, + { + name: "save snapshot fail", + rawWriter: repomocks.NewMockRepositoryWriter(t), + snap: udmrepo.Snapshot{ + Source: "fake-source", + RootObject: udmrepo.ObjectMetadata{ID: "I123456"}, + }, + rawWriterRetErr: errors.New("fake-save-error"), + setWriterMock: true, + expectedErr: "error saving snapshot: error putting manifest: fake-save-error", + }, + { + name: "succeed", + rawWriter: repomocks.NewMockRepositoryWriter(t), + snap: udmrepo.Snapshot{ + Source: "fake-source", + RootObject: udmrepo.ObjectMetadata{ID: "I123456"}, + }, + rawWriterRetID: manifest.ID("fake-manifest-id"), + setWriterMock: true, + expectedID: udmrepo.ID("fake-manifest-id"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + kr := &kopiaRepository{} + + if tc.rawWriter != nil { + if tc.setWriterMock { + tc.rawWriter.On("PutManifest", mock.Anything, mock.Anything, mock.Anything).Return(tc.rawWriterRetID, tc.rawWriterRetErr) + } + kr.rawWriter = tc.rawWriter + } + + id, err := kr.SaveSnapshot(t.Context(), tc.snap) + + if tc.expectedErr == "" { + require.NoError(t, err) + assert.Equal(t, tc.expectedID, id) + } else { + assert.EqualError(t, err, tc.expectedErr) + } + }) + } +} + +func TestGetSnapshot(t *testing.T) { + expectedTime := time.Now() + rawObjID, _ := object.ParseID("I123456") + + mockMani := &snapshot.Manifest{ + Source: snapshot.SourceInfo{Path: "fake-source"}, + Description: "fake-desc", + StartTime: fs.UTCTimestampFromTime(expectedTime), + EndTime: fs.UTCTimestampFromTime(expectedTime.Add(time.Minute)), + RootEntry: &snapshot.DirEntry{ + ObjectID: rawObjID, + }, + Tags: map[string]string{"tag1": "val1"}, + } + + testCases := []struct { + name string + rawRepo *repomocks.MockRepository + snapshotID udmrepo.ID + rawRepoRetErr error + setRepoMock bool + expectedErr string + expectedSnap udmrepo.Snapshot + }{ + { + name: "get snapshot fail", + rawRepo: repomocks.NewMockRepository(t), + snapshotID: udmrepo.ID("fake-id"), + rawRepoRetErr: errors.New("fake-get-error"), + setRepoMock: true, + expectedErr: "error getting snapshot manifest: unable to find manifest entries: fake-get-error", + }, + { + name: "succeed", + rawRepo: repomocks.NewMockRepository(t), + snapshotID: udmrepo.ID("fake-id"), + setRepoMock: true, + expectedSnap: udmrepo.Snapshot{ + Source: "fake-source", + Description: "fake-desc", + StartTime: mockMani.StartTime.ToTime(), + EndTime: mockMani.EndTime.ToTime(), + RootObject: udmrepo.ObjectMetadata{ + ID: udmrepo.ID("I123456"), + Type: udmrepo.ObjectDataTypeMetadata, + Size: mockMani.RootEntry.FileSize, + ModTime: mockMani.RootEntry.ModTime.ToTime(), + Permissions: int(mockMani.RootEntry.Permissions), + UserID: mockMani.RootEntry.UserID, + GroupID: mockMani.RootEntry.GroupID, + }, + Tags: map[string]string{"tag1": "val1"}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + kr := &kopiaRepository{} + + if tc.rawRepo != nil { + if tc.setRepoMock { + tc.rawRepo.On("GetManifest", mock.Anything, mock.Anything, mock.Anything).Return(&manifest.EntryMetadata{ + Labels: map[string]string{ + manifest.TypeLabelKey: snapshot.ManifestType, + }, + }, tc.rawRepoRetErr).Run(func(args mock.Arguments) { + if tc.rawRepoRetErr == nil { + payload := args.Get(2) + if ptr, ok := payload.(*snapshot.Manifest); ok { + *ptr = *mockMani + } else { + b, _ := json.Marshal(mockMani) + json.Unmarshal(b, payload) + } + } + }) + } + kr.rawRepo = tc.rawRepo + } + + snap, err := kr.GetSnapshot(t.Context(), tc.snapshotID) + + if tc.expectedErr == "" { + require.NoError(t, err) + assert.Equal(t, tc.expectedSnap, snap) + } else { + assert.EqualError(t, err, tc.expectedErr) + } + }) + } +} + +func TestDeleteSnapshot(t *testing.T) { + expectedTime := time.Now() + rawObjID, _ := object.ParseID("I123456") + + mockMani := &snapshot.Manifest{ + Source: snapshot.SourceInfo{Path: "fake-source"}, + Description: "fake-desc", + StartTime: fs.UTCTimestampFromTime(expectedTime), + EndTime: fs.UTCTimestampFromTime(expectedTime.Add(time.Minute)), + RootEntry: &snapshot.DirEntry{ + ObjectID: rawObjID, + }, + Tags: map[string]string{"tag1": "val1"}, + } + + testCases := []struct { + name string + rawRepo *repomocks.MockRepository + rawWriter *repomocks.MockRepositoryWriter + snapshotID udmrepo.ID + rawRepoRetErr error + rawWriterRetErr error + setRepoMock bool + setWriterMock bool + expectedErr string + }{ + { + name: "get snapshot fail", + rawRepo: repomocks.NewMockRepository(t), + snapshotID: udmrepo.ID("fake-id"), + rawRepoRetErr: errors.New("fake-get-error"), + setRepoMock: true, + expectedErr: "error getting snapshot: error getting snapshot manifest: unable to find manifest entries: fake-get-error", + }, + { + name: "delete manifest fail", + rawRepo: repomocks.NewMockRepository(t), + rawWriter: repomocks.NewMockRepositoryWriter(t), + snapshotID: udmrepo.ID("fake-id"), + rawWriterRetErr: errors.New("fake-delete-error"), + setRepoMock: true, + setWriterMock: true, + expectedErr: "error to delete manifest: fake-delete-error", + }, + { + name: "succeed", + rawRepo: repomocks.NewMockRepository(t), + rawWriter: repomocks.NewMockRepositoryWriter(t), + snapshotID: udmrepo.ID("fake-id"), + setRepoMock: true, + setWriterMock: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + kr := &kopiaRepository{} + + if tc.rawRepo != nil { + if tc.setRepoMock { + tc.rawRepo.On("GetManifest", mock.Anything, mock.Anything, mock.Anything).Return(&manifest.EntryMetadata{ + Labels: map[string]string{ + manifest.TypeLabelKey: snapshot.ManifestType, + }, + }, tc.rawRepoRetErr).Run(func(args mock.Arguments) { + if tc.rawRepoRetErr == nil { + payload := args.Get(2) + if ptr, ok := payload.(*snapshot.Manifest); ok { + *ptr = *mockMani + } else { + b, _ := json.Marshal(mockMani) + json.Unmarshal(b, payload) + } + } + }) + } + kr.rawRepo = tc.rawRepo + } + + if tc.rawWriter != nil { + if tc.setWriterMock { + tc.rawWriter.On("DeleteManifest", mock.Anything, mock.Anything).Return(tc.rawWriterRetErr) + } + kr.rawWriter = tc.rawWriter + } + + err := kr.DeleteSnapshot(t.Context(), tc.snapshotID) + + if tc.expectedErr == "" { + require.NoError(t, err) + } else { + assert.EqualError(t, err, tc.expectedErr) + } + }) + } +} + +func TestListSnapshot(t *testing.T) { + expectedTime := time.Now() + rawObjID, _ := object.ParseID("I123456") + + mockMani := &snapshot.Manifest{ + Source: snapshot.SourceInfo{Path: "fake-source"}, + Description: "fake-desc", + StartTime: fs.UTCTimestampFromTime(expectedTime), + EndTime: fs.UTCTimestampFromTime(expectedTime.Add(time.Minute)), + RootEntry: &snapshot.DirEntry{ + ObjectID: rawObjID, + FileSize: 100, + ModTime: fs.UTCTimestampFromTime(expectedTime), + Permissions: 0o644, + UserID: 1000, + GroupID: 1000, + }, + Tags: map[string]string{"tag1": "val1"}, + } + + testCases := []struct { + name string + rawRepo *repomocks.MockRepository + source string + findRetErr error + setRepoMock bool + expectedErr string + expectedSnaps []udmrepo.Snapshot + }{ + { + name: "find manifest fail", + rawRepo: repomocks.NewMockRepository(t), + source: "fake-source", + findRetErr: errors.New("fake-find-error"), + setRepoMock: true, + expectedErr: "error listing snapshot manifest for source fake-source: unable to find manifest entries: fake-find-error", + }, + { + name: "succeed", + rawRepo: repomocks.NewMockRepository(t), + source: "fake-source", + setRepoMock: true, + expectedSnaps: []udmrepo.Snapshot{ + { + Source: "fake-source", + Description: "fake-desc", + StartTime: mockMani.StartTime.ToTime(), + EndTime: mockMani.EndTime.ToTime(), + RootObject: udmrepo.ObjectMetadata{ + ID: udmrepo.ID("I123456"), + Type: udmrepo.ObjectDataTypeMetadata, + Size: mockMani.RootEntry.FileSize, + ModTime: mockMani.RootEntry.ModTime.ToTime(), + Permissions: int(mockMani.RootEntry.Permissions), + UserID: mockMani.RootEntry.UserID, + GroupID: mockMani.RootEntry.GroupID, + }, + Tags: map[string]string{"tag1": "val1"}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + kr := &kopiaRepository{} + + if tc.rawRepo != nil { + if tc.setRepoMock { + tc.rawRepo.On("FindManifests", mock.Anything, mock.Anything).Return([]*manifest.EntryMetadata{ + { + ID: "fake-id", + Labels: map[string]string{ + manifest.TypeLabelKey: snapshot.ManifestType, + "hostname": udmrepo.GetRepoDomain(), + "username": udmrepo.GetRepoUser(), + "path": tc.source, + }, + }, + }, tc.findRetErr) + + tc.rawRepo.On("GetManifest", mock.Anything, mock.Anything, mock.Anything).Return(&manifest.EntryMetadata{ + Labels: map[string]string{ + manifest.TypeLabelKey: snapshot.ManifestType, + }, + }, nil).Run(func(args mock.Arguments) { + payload := args.Get(2) + if ptr, ok := payload.(*snapshot.Manifest); ok { + *ptr = *mockMani + } else { + b, _ := json.Marshal(mockMani) + json.Unmarshal(b, payload) + } + }).Maybe() + } + kr.rawRepo = tc.rawRepo + } + + snaps, err := kr.ListSnapshot(t.Context(), tc.source) + + if tc.expectedErr == "" { + require.NoError(t, err) + assert.Equal(t, tc.expectedSnaps, snaps) + } else { + assert.EqualError(t, err, tc.expectedErr) + } + }) + } +} diff --git a/pkg/repository/udmrepo/mocks/BackupRepo.go b/pkg/repository/udmrepo/mocks/BackupRepo.go index e495b6b00..623c4d70d 100644 --- a/pkg/repository/udmrepo/mocks/BackupRepo.go +++ b/pkg/repository/udmrepo/mocks/BackupRepo.go @@ -562,6 +562,74 @@ func (_c *BackupRepo_GetSnapshot_Call) RunAndReturn(run func(ctx context.Context return _c } +// ListSnapshot provides a mock function for the type BackupRepo +func (_mock *BackupRepo) ListSnapshot(ctx context.Context, source string) ([]udmrepo.Snapshot, error) { + ret := _mock.Called(ctx, source) + + if len(ret) == 0 { + panic("no return value specified for ListSnapshot") + } + + var r0 []udmrepo.Snapshot + var r1 error + if returnFunc, ok := ret.Get(0).(func(context.Context, string) ([]udmrepo.Snapshot, error)); ok { + return returnFunc(ctx, source) + } + if returnFunc, ok := ret.Get(0).(func(context.Context, string) []udmrepo.Snapshot); ok { + r0 = returnFunc(ctx, source) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]udmrepo.Snapshot) + } + } + if returnFunc, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = returnFunc(ctx, source) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// BackupRepo_ListSnapshot_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListSnapshot' +type BackupRepo_ListSnapshot_Call struct { + *mock.Call +} + +// ListSnapshot is a helper method to define mock.On call +// - ctx context.Context +// - source string +func (_e *BackupRepo_Expecter) ListSnapshot(ctx interface{}, source interface{}) *BackupRepo_ListSnapshot_Call { + return &BackupRepo_ListSnapshot_Call{Call: _e.mock.On("ListSnapshot", ctx, source)} +} + +func (_c *BackupRepo_ListSnapshot_Call) Run(run func(ctx context.Context, source string)) *BackupRepo_ListSnapshot_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 context.Context + if args[0] != nil { + arg0 = args[0].(context.Context) + } + var arg1 string + if args[1] != nil { + arg1 = args[1].(string) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *BackupRepo_ListSnapshot_Call) Return(snapshots []udmrepo.Snapshot, err error) *BackupRepo_ListSnapshot_Call { + _c.Call.Return(snapshots, err) + return _c +} + +func (_c *BackupRepo_ListSnapshot_Call) RunAndReturn(run func(ctx context.Context, source string) ([]udmrepo.Snapshot, error)) *BackupRepo_ListSnapshot_Call { + _c.Call.Return(run) + return _c +} + // NewObjectWriter provides a mock function for the type BackupRepo func (_mock *BackupRepo) NewObjectWriter(ctx context.Context, opt udmrepo.ObjectWriteOptions) (udmrepo.ObjectWriter, error) { ret := _mock.Called(ctx, opt) diff --git a/pkg/repository/udmrepo/repo.go b/pkg/repository/udmrepo/repo.go index ff60c71f5..7b47eb735 100644 --- a/pkg/repository/udmrepo/repo.go +++ b/pkg/repository/udmrepo/repo.go @@ -98,7 +98,7 @@ type Snapshot struct { StartTime time.Time EndTime time.Time Tags map[string]string - RootObject ID + RootObject ObjectMetadata } // BackupRepoService is used to initialize, open or maintain a backup repository @@ -181,6 +181,9 @@ type BackupRepo interface { // DeleteSnapshot deletes a repo snapshot DeleteSnapshot(ctx context.Context, id ID) error + // ListSnapshot lists all snapshots in repo for the given source + ListSnapshot(ctx context.Context, source string) ([]Snapshot, error) + // Close closes the backup repository Close(ctx context.Context) error }