From 33e3c9589b7dceb9ba8fcdc028d69c9f912b2a2f Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Fri, 22 Dec 2023 14:59:27 +0800 Subject: [PATCH 1/2] issue 7244: delete incomplete snapshot automatically for kopia uploader Signed-off-by: Lyndon-Li --- changelogs/unreleased/7245-Lyndon-Li | 1 + pkg/uploader/kopia/snapshot.go | 6 ++++++ pkg/uploader/kopia/snapshot_test.go | 1 + pkg/uploader/mocks/policy.go | 9 +++++---- 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 changelogs/unreleased/7245-Lyndon-Li diff --git a/changelogs/unreleased/7245-Lyndon-Li b/changelogs/unreleased/7245-Lyndon-Li new file mode 100644 index 000000000..edb945ec3 --- /dev/null +++ b/changelogs/unreleased/7245-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #7244. By the end of the upload, check the outstanding incomplete snapshots and delete them by calling ApplyRetentionPolicy \ No newline at end of file diff --git a/pkg/uploader/kopia/snapshot.go b/pkg/uploader/kopia/snapshot.go index d87404a36..cf6c2b80f 100644 --- a/pkg/uploader/kopia/snapshot.go +++ b/pkg/uploader/kopia/snapshot.go @@ -44,6 +44,7 @@ import ( ) // All function mainly used to make testing more convenient +var applyRetentionPolicyFunc = policy.ApplyRetentionPolicy var treeForSourceFunc = policy.TreeForSource var setPolicyFunc = policy.SetPolicy var saveSnapshotFunc = snapshot.SaveSnapshot @@ -277,6 +278,11 @@ func SnapshotSource( return "", 0, errors.Wrapf(err, "Failed to save kopia manifest %v", manifest.ID) } + _, err = applyRetentionPolicyFunc(ctx, rep, sourceInfo, true) + if err != nil { + return "", 0, errors.Wrapf(err, "Failed to apply kopia retention policy for si %v", sourceInfo) + } + if err = rep.Flush(ctx); err != nil { return "", 0, errors.Wrapf(err, "Failed to flush kopia repository") } diff --git a/pkg/uploader/kopia/snapshot_test.go b/pkg/uploader/kopia/snapshot_test.go index 34bc530b2..62c416029 100644 --- a/pkg/uploader/kopia/snapshot_test.go +++ b/pkg/uploader/kopia/snapshot_test.go @@ -60,6 +60,7 @@ func injectSnapshotFuncs() *snapshotMockes { repoWriterMock: &repomocks.RepositoryWriter{}, } + applyRetentionPolicyFunc = s.policyMock.ApplyRetentionPolicy setPolicyFunc = s.policyMock.SetPolicy treeForSourceFunc = s.policyMock.TreeForSource loadSnapshotFunc = s.snapshotMock.LoadSnapshot diff --git a/pkg/uploader/mocks/policy.go b/pkg/uploader/mocks/policy.go index 3c1dcdd78..ef091c518 100644 --- a/pkg/uploader/mocks/policy.go +++ b/pkg/uploader/mocks/policy.go @@ -19,6 +19,7 @@ package mocks import ( "context" + "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot/policy" "github.com/stretchr/testify/mock" @@ -56,15 +57,15 @@ func (_m *Policy) TreeForSource(ctx context.Context, rep repo.Repository, si sna } // ApplyRetentionPolicy provides a mock function with given fields: ctx, rep, sourceInfo, reallyDelete -func (_m *Policy) ApplyRetentionPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, reallyDelete bool) ([]*snapshot.Manifest, error) { +func (_m *Policy) ApplyRetentionPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, reallyDelete bool) ([]manifest.ID, error) { ret := _m.Called(ctx, rep, sourceInfo, reallyDelete) - var r0 []*snapshot.Manifest - if rf, ok := ret.Get(0).(func(context.Context, repo.RepositoryWriter, snapshot.SourceInfo, bool) []*snapshot.Manifest); ok { + var r0 []manifest.ID + if rf, ok := ret.Get(0).(func(context.Context, repo.RepositoryWriter, snapshot.SourceInfo, bool) []manifest.ID); ok { r0 = rf(ctx, rep, sourceInfo, reallyDelete) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]*snapshot.Manifest) + r0 = ret.Get(0).([]manifest.ID) } } From 5ff8a4f2a36e7adce2374bbe7c649e463f63bc77 Mon Sep 17 00:00:00 2001 From: Lyndon-Li Date: Fri, 22 Dec 2023 16:57:58 +0800 Subject: [PATCH 2/2] Issue 7244: delete incomplete snapshot automatically for kopia uploader Signed-off-by: Lyndon-Li --- changelogs/unreleased/{7245-Lyndon-Li => 7247-Lyndon-Li} | 0 pkg/uploader/mocks/policy.go | 9 ++++----- 2 files changed, 4 insertions(+), 5 deletions(-) rename changelogs/unreleased/{7245-Lyndon-Li => 7247-Lyndon-Li} (100%) diff --git a/changelogs/unreleased/7245-Lyndon-Li b/changelogs/unreleased/7247-Lyndon-Li similarity index 100% rename from changelogs/unreleased/7245-Lyndon-Li rename to changelogs/unreleased/7247-Lyndon-Li diff --git a/pkg/uploader/mocks/policy.go b/pkg/uploader/mocks/policy.go index ef091c518..3c1dcdd78 100644 --- a/pkg/uploader/mocks/policy.go +++ b/pkg/uploader/mocks/policy.go @@ -19,7 +19,6 @@ package mocks import ( "context" - "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot/policy" "github.com/stretchr/testify/mock" @@ -57,15 +56,15 @@ func (_m *Policy) TreeForSource(ctx context.Context, rep repo.Repository, si sna } // ApplyRetentionPolicy provides a mock function with given fields: ctx, rep, sourceInfo, reallyDelete -func (_m *Policy) ApplyRetentionPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, reallyDelete bool) ([]manifest.ID, error) { +func (_m *Policy) ApplyRetentionPolicy(ctx context.Context, rep repo.RepositoryWriter, sourceInfo snapshot.SourceInfo, reallyDelete bool) ([]*snapshot.Manifest, error) { ret := _m.Called(ctx, rep, sourceInfo, reallyDelete) - var r0 []manifest.ID - if rf, ok := ret.Get(0).(func(context.Context, repo.RepositoryWriter, snapshot.SourceInfo, bool) []manifest.ID); ok { + var r0 []*snapshot.Manifest + if rf, ok := ret.Get(0).(func(context.Context, repo.RepositoryWriter, snapshot.SourceInfo, bool) []*snapshot.Manifest); ok { r0 = rf(ctx, rep, sourceInfo, reallyDelete) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]manifest.ID) + r0 = ret.Get(0).([]*snapshot.Manifest) } }