From 29b2cd1883faf4f70dd8508224ec9673429be6d4 Mon Sep 17 00:00:00 2001 From: danfengl Date: Fri, 26 Nov 2021 10:29:53 +0000 Subject: [PATCH] Add backup deletion e2e test Test case description is "Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", in this test, only resource backup objects are target for verifition, restic repo verification is not included in this PR, and snapshot verification will be in later PR Signed-off-by: danfengl --- changelogs/unreleased/4401-danfengliu | 1 + go.mod | 11 +- go.sum | 27 ++- test/e2e/backups/deletion.go | 159 +++++++++++++++ test/e2e/e2e_suite_test.go | 2 + test/e2e/util/providers/aws_utils.go | 141 +++++++++++++ test/e2e/util/providers/azure_utils.go | 258 ++++++++++++++++++++++++ test/e2e/util/providers/common.go | 105 ++++++++++ test/e2e/util/providers/gcloud_utils.go | 99 +++++++++ test/e2e/util/velero/velero_utils.go | 39 ++++ 10 files changed, 836 insertions(+), 6 deletions(-) create mode 100644 changelogs/unreleased/4401-danfengliu create mode 100644 test/e2e/backups/deletion.go create mode 100644 test/e2e/util/providers/aws_utils.go create mode 100644 test/e2e/util/providers/azure_utils.go create mode 100644 test/e2e/util/providers/common.go create mode 100644 test/e2e/util/providers/gcloud_utils.go diff --git a/changelogs/unreleased/4401-danfengliu b/changelogs/unreleased/4401-danfengliu new file mode 100644 index 000000000..e0982ae12 --- /dev/null +++ b/changelogs/unreleased/4401-danfengliu @@ -0,0 +1 @@ +Add backup deletion e2e test \ No newline at end of file diff --git a/go.mod b/go.mod index b55a85266..972a2a9ab 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,10 @@ module github.com/vmware-tanzu/velero go 1.17 require ( + cloud.google.com/go/storage v1.10.0 + github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-sdk-for-go v42.0.0+incompatible + github.com/Azure/azure-storage-blob-go v0.14.0 github.com/Azure/go-autorest/autorest v0.11.21 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/to v0.3.0 @@ -14,12 +17,12 @@ require ( github.com/gobwas/glob v0.2.3 github.com/gofrs/uuid v3.2.0+incompatible github.com/golang/protobuf v1.5.2 - github.com/google/uuid v1.1.2 + github.com/google/uuid v1.2.0 github.com/hashicorp/go-hclog v0.12.0 github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 github.com/joho/godotenv v1.3.0 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 - github.com/onsi/ginkgo v1.16.4 + github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.16.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 @@ -32,6 +35,7 @@ require ( github.com/vmware-tanzu/crash-diagnostics v0.3.7 golang.org/x/mod v0.4.2 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 + google.golang.org/api v0.56.0 google.golang.org/grpc v1.40.0 k8s.io/api v0.22.2 k8s.io/apiextensions-apiserver v0.22.2 @@ -67,6 +71,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -75,6 +80,7 @@ require ( github.com/json-iterator/go v1.1.11 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mattn/go-colorable v0.1.9 // indirect + github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -90,6 +96,7 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/stretchr/objx v0.2.0 // indirect github.com/vladimirvivien/gexe v0.1.1 // indirect + go.opencensus.io v0.23.0 // indirect go.starlark.net v0.0.0-20201006213952-227f4aabceb5 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect diff --git a/go.sum b/go.sum index 5084b6c5b..525808f44 100644 --- a/go.sum +++ b/go.sum @@ -43,10 +43,15 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v42.0.0+incompatible h1:yz6sFf5bHZ+gEOQVuK5JhPqTTAmv+OvSLSaqgzqaCwY= github.com/Azure/azure-sdk-for-go v42.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= @@ -320,9 +325,11 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -342,10 +349,12 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= @@ -435,6 +444,8 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -458,6 +469,8 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -505,7 +518,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -520,8 +532,9 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -666,6 +679,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= @@ -770,6 +784,7 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -854,6 +869,7 @@ golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -877,6 +893,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1027,6 +1044,7 @@ google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtuk google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.56.0 h1:08F9XVYTLOGeSQb3xI9C0gXMuQanhdGed0cWFhDozbI= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1138,8 +1156,9 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go new file mode 100644 index 000000000..477bf227d --- /dev/null +++ b/test/e2e/backups/deletion.go @@ -0,0 +1,159 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package backups + +import ( + "context" + "flag" + "fmt" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + . "github.com/vmware-tanzu/velero/test/e2e/util/providers" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +const ( + deletionTest = "deletion-workload" +) + +// Test backup and restore of Kibishi using restic + +func Backup_deletion_with_snapshots() { + backup_deletion_test(true) +} + +func Backup_deletion_with_restic() { + backup_deletion_test(false) +} +func backup_deletion_test(useVolumeSnapshots bool) { + var ( + backupName string + ) + + client, err := NewTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests") + + BeforeEach(func() { + if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { + Skip("Volume snapshots not supported on kind") + } + var err error + flag.Parse() + UUIDgen, err = uuid.NewRandom() + Expect(err).To(Succeed()) + if VeleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + Expect(err).To(Succeed()) + } + }) + + When("kibishii is the sample workload", func() { + It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() { + backupName = "backup-" + UUIDgen.String() + Expect(runBackupDeletionTests(client, VeleroCfg.VeleroCLI, VeleroCfg.CloudProvider, VeleroCfg.VeleroNamespace, backupName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig)).To(Succeed(), + "Failed to run backup deletion test") + }) + }) +} + +// runUpgradeTests runs upgrade test on the provider by kibishii. +func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNamespace, backupName, backupLocation string, + useVolumeSnapshots bool, registryCredentialFile, bslPrefix, bslConfig string) error { + + oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + + if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest) + } + defer func() { + if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil { + fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest)) + } + }() + + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile); err != nil { + return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest) + } + err := ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1) + if err != nil { + return err + } + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, deletionTest, backupLocation, useVolumeSnapshots); err != nil { + // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command + // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case + VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName) + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", deletionTest) + } + + if providerName == "vsphere" && useVolumeSnapshots { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + fmt.Println("Waiting for vSphere uploads to complete") + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, deletionTest); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + } + err = ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + if err != nil { + return err + } + err = DeleteBackupResource(context.Background(), veleroCLI, backupName) + if err != nil { + return err + } + err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) + if err != nil { + fmt.Println(errors.Wrapf(err, "Failed to get object from bucket %q", backupName)) + return err + } + backupName = "backup-1-" + UUIDgen.String() + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, deletionTest, backupLocation, useVolumeSnapshots); err != nil { + // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command + // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case + VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName) + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", deletionTest) + } + err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + if err != nil { + fmt.Println(errors.Wrapf(err, "Failed to delete object in bucket %q", backupName)) + return err + } + err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) + if err != nil { + return err + } + err = DeleteBackupResource(context.Background(), veleroCLI, backupName) + if err != nil { + fmt.Println(errors.Wrapf(err, "|| UNEXPECTED || - Failed to delete backup %q", backupName)) + return err + } + fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n") + return nil +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 93876044a..703257c0d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -26,6 +26,7 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/backup" + . "github.com/vmware-tanzu/velero/test/e2e/backups" . "github.com/vmware-tanzu/velero/test/e2e/basic" . "github.com/vmware-tanzu/velero/test/e2e/resource-filtering" . "github.com/vmware-tanzu/velero/test/e2e/scale" @@ -86,6 +87,7 @@ var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources) var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) +var _ = Describe("[Backups][Deletion] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", Backup_deletion_with_restic) func TestE2e(t *testing.T) { // Skip running E2E tests when running only "short" tests because: diff --git a/test/e2e/util/providers/aws_utils.go b/test/e2e/util/providers/aws_utils.go new file mode 100644 index 000000000..3e64b2b3a --- /dev/null +++ b/test/e2e/util/providers/aws_utils.go @@ -0,0 +1,141 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/pkg/errors" + + "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" +) + +type AWSStorage string + +func (s AWSStorage) ListItems(client *s3.S3, objectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + res, err := client.ListObjectsV2(objectsV2Input) + if err != nil { + return nil, err + } + + return res, nil +} + +func (s AWSStorage) DeleteItem(client *s3.S3, deleteObjectV2Input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + res, err := client.DeleteObject(deleteObjectV2Input) + if err != nil { + return nil, err + } + fmt.Println(res) + return res, nil +} +func (s AWSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { + config := flag.NewMap() + config.Set(bslConfig) + region := config.Data()["region"] + objectsInput := s3.ListObjectsV2Input{} + objectsInput.Bucket = aws.String(bslBucket) + objectsInput.Delimiter = aws.String("/") + s3url := "" + if bslPrefix != "" { + objectsInput.Prefix = aws.String(bslPrefix) + } + s3Config := &aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + } + if region == "minio" { + s3url = config.Data()["s3Url"] + s3Config = &aws.Config{ + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + Endpoint: aws.String(s3url), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + } + + sess, err := session.NewSession(s3Config) + + if err != nil { + return false, errors.Wrapf(err, "Failed to create AWS session") + } + svc := s3.New(sess) + + bucketObjects, err := s.ListItems(svc, &objectsInput) + if err != nil { + return false, errors.Wrapf(err, "Couldn't retrieve bucket items") + } + + for _, item := range bucketObjects.Contents { + fmt.Println(*item) + } + var backupNameInStorage string + for _, item := range bucketObjects.CommonPrefixes { + backupNameInStorage = strings.TrimPrefix(*item.Prefix, strings.Trim(bslPrefix, "/")+"/") + fmt.Println(backupNameInStorage) + if strings.Contains(backupNameInStorage, backupObject) { + fmt.Printf("Backup %s was found under prefix %s \n", backupObject, bslPrefix) + return true, nil + } + } + fmt.Printf("Backup %s was not found under prefix %s \n", backupObject, bslPrefix) + return false, nil +} + +func (s AWSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error { + config := flag.NewMap() + config.Set(bslConfig) + region := config.Data()["region"] + s3url := "" + s3Config := &aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + } + if region == "minio" { + s3url = config.Data()["s3Url"] + s3Config = &aws.Config{ + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + Endpoint: aws.String(s3url), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + } + sess, err := session.NewSession(s3Config) + if err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + svc := s3.New(sess) + fullPrefix := strings.Trim(bslPrefix, "/") + "/" + strings.Trim(backupObject, "/") + "/" + iter := s3manager.NewDeleteListIterator(svc, &s3.ListObjectsInput{ + Bucket: aws.String(bslBucket), + Prefix: aws.String(fullPrefix), + }) + + if err := s3manager.NewBatchDeleteWithClient(svc).Delete(aws.BackgroundContext(), iter); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + fmt.Printf("Deleted object(s) from bucket: %s %s \n", bslBucket, fullPrefix) + return nil +} diff --git a/test/e2e/util/providers/azure_utils.go b/test/e2e/util/providers/azure_utils.go new file mode 100644 index 000000000..d9ee4c6ca --- /dev/null +++ b/test/e2e/util/providers/azure_utils.go @@ -0,0 +1,258 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "log" + "net/url" + "os" + "strings" + + "github.com/Azure/azure-pipeline-go/pipeline" + storagemgmt "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/azure/auth" + "github.com/joho/godotenv" + "github.com/pkg/errors" + "golang.org/x/net/context" + + "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" +) + +type AzureStorage string + +const ( + subscriptionIDEnvVar = "AZURE_SUBSCRIPTION_ID" + cloudNameEnvVar = "AZURE_CLOUD_NAME" + resourceGroupEnvVar = "AZURE_RESOURCE_GROUP" + storageAccountKey = "AZURE_STORAGE_ACCOUNT_ACCESS_KEY" + storageAccount = "storageAccount" + subscriptionID = "subscriptionId" + resourceGroup = "resourceGroup" +) + +func getStorageCredential(cloudCredentialsFile, bslConfig string) (string, string, error) { + config := flag.NewMap() + config.Set(bslConfig) + accountName := config.Data()[storageAccount] + // Account name must be provided in config + if len(accountName) == 0 { + return "", "", errors.New("Please provide bucket as Azure account name ") + } + subscriptionID := config.Data()[subscriptionID] + resourceGroupCfg := config.Data()[resourceGroup] + accountKey, err := getStorageAccountKey(cloudCredentialsFile, accountName, subscriptionID, resourceGroupCfg) + if err != nil { + return "", "", errors.Wrapf(err, "Fail to get storage key of bucket %s", accountName) + } + return accountName, accountKey, nil +} +func loadCredentialsIntoEnv(credentialsFile string) error { + if credentialsFile == "" { + return nil + } + + if err := godotenv.Overload(credentialsFile); err != nil { + return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile) + } + return nil +} +func parseAzureEnvironment(cloudName string) (*azure.Environment, error) { + if cloudName == "" { + fmt.Println("cloudName is empty") + return &azure.PublicCloud, nil + } + + env, err := azure.EnvironmentFromName(cloudName) + return &env, errors.WithStack(err) +} +func getStorageAccountKey(credentialsFile, accountName, subscriptionID, resourceGroupCfg string) (string, error) { + if err := loadCredentialsIntoEnv(credentialsFile); err != nil { + return "", err + } + storageKey := os.Getenv(storageAccountKey) + if storageKey != "" { + return storageKey, nil + } + if os.Getenv(cloudNameEnvVar) == "" { + return "", errors.New("Credential file should contain AZURE_CLOUD_NAME") + } + var resourceGroup string + if os.Getenv(resourceGroupEnvVar) == "" { + if resourceGroupCfg == "" { + return "", errors.New("Credential file should contain AZURE_RESOURCE_GROUP or AZURE_STORAGE_ACCOUNT_ACCESS_KEY") + } else { + resourceGroup = resourceGroupCfg + } + } else { + resourceGroup = os.Getenv(resourceGroupEnvVar) + } + // get Azure cloud from AZURE_CLOUD_NAME, if it exists. If the env var does not + // exist, parseAzureEnvironment will return azure.PublicCloud. + env, err := parseAzureEnvironment(os.Getenv(cloudNameEnvVar)) + if err != nil { + return "", errors.Wrap(err, "unable to parse azure cloud name environment variable") + } + + // get subscription ID from object store config or AZURE_SUBSCRIPTION_ID environment variable + if subscriptionID == "" { + return "", errors.New("azure subscription ID not found in object store's config or in environment variable") + } + + authorizer, err := auth.NewAuthorizerFromEnvironment() + if err != nil { + return "", errors.Wrap(err, "error getting authorizer from environment") + } + + // get storageAccountsClient + storageAccountsClient := storagemgmt.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID) + storageAccountsClient.Authorizer = authorizer + + // get storage key + res, err := storageAccountsClient.ListKeys(context.TODO(), resourceGroup, accountName, storagemgmt.Kerb) + if err != nil { + return "", errors.WithStack(err) + } + if res.Keys == nil || len(*res.Keys) == 0 { + return "", errors.New("No storage keys found") + } + + for _, key := range *res.Keys { + // uppercase both strings for comparison because the ListKeys call returns e.g. "FULL" but + // the storagemgmt.Full constant in the SDK is defined as "Full". + if strings.EqualFold(string(key.Permissions), string(storagemgmt.Full)) { + storageKey = *key.Value + break + } + } + + if storageKey == "" { + return "", errors.New("No storage key with Full permissions found") + } + + return storageKey, nil +} +func handleErrors(err error) { + if err != nil { + if serr, ok := err.(azblob.StorageError); ok { // This error is a Service-specific + switch serr.ServiceCode() { // Compare serviceCode to ServiceCodeXxx constants + case azblob.ServiceCodeContainerAlreadyExists: + return + } + } + log.Fatal(err) + } +} + +func deleteBlob(p pipeline.Pipeline, accountName, containerName, blobName string) error { + ctx := context.Background() + + URL_BLOB, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", accountName, containerName, blobName)) + if err != nil { + return errors.Wrapf(err, "Fail to url.Parse") + } + blobURL := azblob.NewBlobURL(*URL_BLOB, p) + _, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + return err +} +func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { + accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig) + if err != nil { + log.Fatal("Fail to get : accountName and accountKey, " + err.Error()) + } + + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + log.Fatal("Invalid credentials with error: " + err.Error()) + } + p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + + containerName := bslBucket + + URL, _ := url.Parse( + fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName)) + + containerURL := azblob.NewContainerURL(*URL, p) + + // Create the container, if container is already exist, then do nothing + ctx := context.Background() + _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) + handleErrors(err) + + fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupObject, containerName) + for marker := (azblob.Marker{}); marker.NotDone(); { + listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{}) + if err != nil { + return false, errors.Wrapf(err, "Fail to create gcloud client") + } + marker = listBlob.NextMarker + + for _, blobInfo := range listBlob.Segment.BlobItems { + if strings.Contains(blobInfo.Name, backupObject) { + fmt.Printf("Blob name: %s exist in %s\n", backupObject, blobInfo.Name) + return true, nil + } + } + } + return false, nil +} + +func (s AzureStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error { + ctx := context.Background() + accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig) + if err != nil { + return errors.Wrapf(err, "Fail to get storage account name and key of bucket %s", bslBucket) + } + + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + log.Fatal("Invalid credentials with error: " + err.Error()) + } + p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + + containerName := bslBucket + + URL, _ := url.Parse( + fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName)) + + containerURL := azblob.NewContainerURL(*URL, p) + _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) + handleErrors(err) + + fmt.Println("Listing the blobs in the container:") + for marker := (azblob.Marker{}); marker.NotDone(); { + listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{}) + if err != nil { + return errors.Wrapf(err, "Fail to create gcloud client") + } + + marker = listBlob.NextMarker + for _, blobInfo := range listBlob.Segment.BlobItems { + + if strings.Contains(blobInfo.Name, bslPrefix+backupObject+"/") { + deleteBlob(p, accountName, containerName, blobInfo.Name) + if err != nil { + log.Fatal("Invalid credentials with error: " + err.Error()) + } + fmt.Printf("Deleted blob: %s according to backup resource %s\n", blobInfo.Name, bslPrefix+backupObject+"/") + } + } + } + return nil +} diff --git a/test/e2e/util/providers/common.go b/test/e2e/util/providers/common.go new file mode 100644 index 000000000..aa00065c9 --- /dev/null +++ b/test/e2e/util/providers/common.go @@ -0,0 +1,105 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "strings" + "time" + + "github.com/pkg/errors" +) + +type ObjectsInStorage interface { + IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) + DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error +} + +func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { + fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage %s", backupName, bslPrefix) + exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) + if !exist { + return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected", backupName)) + } + fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket) + return nil +} +func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error { + var err error + var exist bool + fmt.Printf("|| VERIFICATION || - Backup %s should not exist in storage %s", backupName, bslPrefix) + for i := 0; i < retryTimes; i++ { + exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) + if err != nil { + return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store", backupName) + } + if !exist { + fmt.Printf("|| EXPECTED || - Backup %s is not in object store\n", backupName) + return nil + } + time.Sleep(1 * time.Minute) + } + return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion", backupName)) +} +func getProvider(cloudProvider string) (ObjectsInStorage, error) { + var s ObjectsInStorage + switch cloudProvider { + case "aws", "vsphere": + aws := AWSStorage("") + s = &aws + case "gcp": + gcs := GCSStorage("") + s = &gcs + case "azure": + az := AzureStorage("") + s = &az + default: + return nil, errors.New(fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + } + return s, nil +} +func getFullPrefix(bslPrefix, subPrefix string) string { + if bslPrefix == "" { + bslPrefix = subPrefix + "/" + } else { + //subPrefix must have surfix "/", so that objects under it can be listed + bslPrefix = strings.Trim(bslPrefix, "/") + "/" + strings.Trim(subPrefix, "/") + "/" + } + return bslPrefix +} +func IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) (bool, error) { + bslPrefix = getFullPrefix(bslPrefix, subPrefix) + s, err := getProvider(cloudProvider) + if err != nil { + return false, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + } + return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) +} + +func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { + bslPrefix = getFullPrefix(bslPrefix, subPrefix) + fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s", backupName, bslPrefix) + s, err := getProvider(cloudProvider) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + } + err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("Fail to delete %s", bslPrefix)) + } + return nil +} diff --git a/test/e2e/util/providers/gcloud_utils.go b/test/e2e/util/providers/gcloud_utils.go new file mode 100644 index 000000000..fdc01b081 --- /dev/null +++ b/test/e2e/util/providers/gcloud_utils.go @@ -0,0 +1,99 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "strings" + + "cloud.google.com/go/storage" + "github.com/pkg/errors" + "golang.org/x/net/context" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +type GCSStorage string + +func (s GCSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { + q := &storage.Query{ + Prefix: bslPrefix, + } + + ctx := context.Background() + client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile)) + if err != nil { + return false, errors.Wrapf(err, "Fail to create gcloud client") + } + iter := client.Bucket(bslBucket).Objects(context.Background(), q) + for { + obj, err := iter.Next() + if err == iterator.Done { + return false, errors.Wrapf(err, fmt.Sprintf("Backup %s was not found under prefix %s \n", backupObject, bslPrefix)) + } + if err != nil { + return false, errors.WithStack(err) + } + if obj.Name == bslPrefix { + fmt.Println("Ignore GCS prefix itself") + continue + } + if strings.Contains(obj.Name, bslPrefix+backupObject+"/") { + fmt.Printf("Found delete-object %s of %s in bucket %s \n", backupObject, obj.Name, bslBucket) + return true, nil + } + } +} +func (s GCSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error { + q := &storage.Query{ + Prefix: bslPrefix, + } + + ctx := context.Background() + client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile)) + if err != nil { + return errors.Wrapf(err, "Fail to create gcloud client") + } + bucket := client.Bucket(bslBucket) + iter := bucket.Objects(context.Background(), q) + deleted := false + for { + obj, err := iter.Next() + if err == iterator.Done { + fmt.Println(err) + if !deleted { + return errors.New("|| UNEXPECTED ||Backup object is not exist and was not deleted in object store") + } + return nil + } + if err != nil { + return errors.WithStack(err) + } + if obj.Name == bslPrefix { + fmt.Println("Ignore GCS prefix itself") + continue + } + // Only delete folder named as backupObject under prefix + if strings.Contains(obj.Name, bslPrefix+backupObject+"/") { + if err = bucket.Object(obj.Name).Delete(ctx); err != nil { + return errors.Wrapf(err, fmt.Sprintf("Fail to delete object %s in bucket %s", obj.Name, bslBucket)) + } + fmt.Printf("Delete item: %s\n", obj.Name) + deleted = true + } + } +} diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index d32c9f577..a7db0808e 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -41,6 +41,10 @@ import ( veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) +const ( + BackupObjectsPrefix = "backups" +) + var pluginsMatrix = map[string]map[string][]string{ "v1.4": { "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, @@ -388,6 +392,9 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro // installs them in the current Velero installation, skipping over those that are already installed. func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error { plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins) + fmt.Printf("addPlugins cmd =%v\n", addPlugins) + fmt.Printf("provider cmd = %v\n", provider) + fmt.Printf("plugins cmd = %v\n", plugins) if err != nil { return errors.WithMessage(err, "Failed to get plugins") } @@ -396,6 +403,7 @@ func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa stderrBuf := new(bytes.Buffer) installPluginCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "plugin", "add", plugin) + fmt.Printf("installPluginCmd cmd =%v\n", installPluginCmd) installPluginCmd.Stdout = stdoutBuf installPluginCmd.Stderr = stderrBuf @@ -557,3 +565,34 @@ func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) { return tmpfile, nil } +func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName string) error { + args := []string{"backup", "delete", backupName, "--confirm"} + + cmd := exec.CommandContext(ctx, veleroCLI, args...) + fmt.Println("Delete backup Command:" + cmd.String()) + stdout, stderr, err := veleroexec.RunCommand(cmd) + if err != nil { + return errors.Wrapf(err, "Fail to get delete backup, stdout=%s, stderr=%s", stdout, stderr) + } + + output := strings.Replace(stdout, "\n", " ", -1) + fmt.Println("Backup delete command output:" + output) + + args = []string{"backup", "get", backupName} + + retryTimes := 5 + for i := 1; i < retryTimes+1; i++ { + cmd = exec.CommandContext(ctx, veleroCLI, args...) + fmt.Printf("Try %d times to delete backup %s \n", i, cmd.String()) + stdout, stderr, err = veleroexec.RunCommand(cmd) + if err != nil { + if strings.Contains(stderr, "not found") { + fmt.Printf("|| EXPECTED || - Backup %s was deleted successfully according to message %s\n", backupName, stderr) + return nil + } + return errors.Wrapf(err, "Fail to get delete backup, stdout=%s, stderr=%s", stdout, stderr) + } + time.Sleep(1 * time.Minute) + } + return nil +}