diff --git a/changelogs/unreleased/4401-danfengliu b/changelogs/unreleased/4401-danfengliu new file mode 100644 index 000000000..e0982ae12 --- /dev/null +++ b/changelogs/unreleased/4401-danfengliu @@ -0,0 +1 @@ +Add backup deletion e2e test \ No newline at end of file diff --git a/go.mod b/go.mod index b55a85266..972a2a9ab 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,10 @@ module github.com/vmware-tanzu/velero go 1.17 require ( + cloud.google.com/go/storage v1.10.0 + github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-sdk-for-go v42.0.0+incompatible + github.com/Azure/azure-storage-blob-go v0.14.0 github.com/Azure/go-autorest/autorest v0.11.21 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/to v0.3.0 @@ -14,12 +17,12 @@ require ( github.com/gobwas/glob v0.2.3 github.com/gofrs/uuid v3.2.0+incompatible github.com/golang/protobuf v1.5.2 - github.com/google/uuid v1.1.2 + github.com/google/uuid v1.2.0 github.com/hashicorp/go-hclog v0.12.0 github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 github.com/joho/godotenv v1.3.0 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 - github.com/onsi/ginkgo v1.16.4 + github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.16.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 @@ -32,6 +35,7 @@ require ( github.com/vmware-tanzu/crash-diagnostics v0.3.7 golang.org/x/mod v0.4.2 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 + google.golang.org/api v0.56.0 google.golang.org/grpc v1.40.0 k8s.io/api v0.22.2 k8s.io/apiextensions-apiserver v0.22.2 @@ -67,6 +71,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -75,6 +80,7 @@ require ( github.com/json-iterator/go v1.1.11 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mattn/go-colorable v0.1.9 // indirect + github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -90,6 +96,7 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect github.com/stretchr/objx v0.2.0 // indirect github.com/vladimirvivien/gexe v0.1.1 // indirect + go.opencensus.io v0.23.0 // indirect go.starlark.net v0.0.0-20201006213952-227f4aabceb5 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect diff --git a/go.sum b/go.sum index 5084b6c5b..525808f44 100644 --- a/go.sum +++ b/go.sum @@ -43,10 +43,15 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v42.0.0+incompatible h1:yz6sFf5bHZ+gEOQVuK5JhPqTTAmv+OvSLSaqgzqaCwY= github.com/Azure/azure-sdk-for-go v42.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= @@ -320,9 +325,11 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -342,10 +349,12 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= @@ -435,6 +444,8 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -458,6 +469,8 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -505,7 +518,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -520,8 +532,9 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -666,6 +679,7 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= @@ -770,6 +784,7 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -854,6 +869,7 @@ golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -877,6 +893,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1027,6 +1044,7 @@ google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtuk google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.56.0 h1:08F9XVYTLOGeSQb3xI9C0gXMuQanhdGed0cWFhDozbI= google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1138,8 +1156,9 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go new file mode 100644 index 000000000..477bf227d --- /dev/null +++ b/test/e2e/backups/deletion.go @@ -0,0 +1,159 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package backups + +import ( + "context" + "flag" + "fmt" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + . "github.com/vmware-tanzu/velero/test/e2e/util/providers" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +const ( + deletionTest = "deletion-workload" +) + +// Test backup and restore of Kibishi using restic + +func Backup_deletion_with_snapshots() { + backup_deletion_test(true) +} + +func Backup_deletion_with_restic() { + backup_deletion_test(false) +} +func backup_deletion_test(useVolumeSnapshots bool) { + var ( + backupName string + ) + + client, err := NewTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests") + + BeforeEach(func() { + if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { + Skip("Volume snapshots not supported on kind") + } + var err error + flag.Parse() + UUIDgen, err = uuid.NewRandom() + Expect(err).To(Succeed()) + if VeleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + Expect(err).To(Succeed()) + } + }) + + When("kibishii is the sample workload", func() { + It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() { + backupName = "backup-" + UUIDgen.String() + Expect(runBackupDeletionTests(client, VeleroCfg.VeleroCLI, VeleroCfg.CloudProvider, VeleroCfg.VeleroNamespace, backupName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig)).To(Succeed(), + "Failed to run backup deletion test") + }) + }) +} + +// runUpgradeTests runs upgrade test on the provider by kibishii. +func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNamespace, backupName, backupLocation string, + useVolumeSnapshots bool, registryCredentialFile, bslPrefix, bslConfig string) error { + + oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + + if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest) + } + defer func() { + if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil { + fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest)) + } + }() + + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile); err != nil { + return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest) + } + err := ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1) + if err != nil { + return err + } + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, deletionTest, backupLocation, useVolumeSnapshots); err != nil { + // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command + // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case + VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName) + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", deletionTest) + } + + if providerName == "vsphere" && useVolumeSnapshots { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + fmt.Println("Waiting for vSphere uploads to complete") + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, deletionTest); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + } + err = ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + if err != nil { + return err + } + err = DeleteBackupResource(context.Background(), veleroCLI, backupName) + if err != nil { + return err + } + err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) + if err != nil { + fmt.Println(errors.Wrapf(err, "Failed to get object from bucket %q", backupName)) + return err + } + backupName = "backup-1-" + UUIDgen.String() + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, deletionTest, backupLocation, useVolumeSnapshots); err != nil { + // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command + // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case + VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName) + return errors.Wrapf(err, "Failed to backup kibishii namespace %s", deletionTest) + } + err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + if err != nil { + fmt.Println(errors.Wrapf(err, "Failed to delete object in bucket %q", backupName)) + return err + } + err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) + if err != nil { + return err + } + err = DeleteBackupResource(context.Background(), veleroCLI, backupName) + if err != nil { + fmt.Println(errors.Wrapf(err, "|| UNEXPECTED || - Failed to delete backup %q", backupName)) + return err + } + fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n") + return nil +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 93876044a..703257c0d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -26,6 +26,7 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/backup" + . "github.com/vmware-tanzu/velero/test/e2e/backups" . "github.com/vmware-tanzu/velero/test/e2e/basic" . "github.com/vmware-tanzu/velero/test/e2e/resource-filtering" . "github.com/vmware-tanzu/velero/test/e2e/scale" @@ -86,6 +87,7 @@ var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources) var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) +var _ = Describe("[Backups][Deletion] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", Backup_deletion_with_restic) func TestE2e(t *testing.T) { // Skip running E2E tests when running only "short" tests because: diff --git a/test/e2e/util/providers/aws_utils.go b/test/e2e/util/providers/aws_utils.go new file mode 100644 index 000000000..3e64b2b3a --- /dev/null +++ b/test/e2e/util/providers/aws_utils.go @@ -0,0 +1,141 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/pkg/errors" + + "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" +) + +type AWSStorage string + +func (s AWSStorage) ListItems(client *s3.S3, objectsV2Input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + res, err := client.ListObjectsV2(objectsV2Input) + if err != nil { + return nil, err + } + + return res, nil +} + +func (s AWSStorage) DeleteItem(client *s3.S3, deleteObjectV2Input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + res, err := client.DeleteObject(deleteObjectV2Input) + if err != nil { + return nil, err + } + fmt.Println(res) + return res, nil +} +func (s AWSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { + config := flag.NewMap() + config.Set(bslConfig) + region := config.Data()["region"] + objectsInput := s3.ListObjectsV2Input{} + objectsInput.Bucket = aws.String(bslBucket) + objectsInput.Delimiter = aws.String("/") + s3url := "" + if bslPrefix != "" { + objectsInput.Prefix = aws.String(bslPrefix) + } + s3Config := &aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + } + if region == "minio" { + s3url = config.Data()["s3Url"] + s3Config = &aws.Config{ + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + Endpoint: aws.String(s3url), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + } + + sess, err := session.NewSession(s3Config) + + if err != nil { + return false, errors.Wrapf(err, "Failed to create AWS session") + } + svc := s3.New(sess) + + bucketObjects, err := s.ListItems(svc, &objectsInput) + if err != nil { + return false, errors.Wrapf(err, "Couldn't retrieve bucket items") + } + + for _, item := range bucketObjects.Contents { + fmt.Println(*item) + } + var backupNameInStorage string + for _, item := range bucketObjects.CommonPrefixes { + backupNameInStorage = strings.TrimPrefix(*item.Prefix, strings.Trim(bslPrefix, "/")+"/") + fmt.Println(backupNameInStorage) + if strings.Contains(backupNameInStorage, backupObject) { + fmt.Printf("Backup %s was found under prefix %s \n", backupObject, bslPrefix) + return true, nil + } + } + fmt.Printf("Backup %s was not found under prefix %s \n", backupObject, bslPrefix) + return false, nil +} + +func (s AWSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error { + config := flag.NewMap() + config.Set(bslConfig) + region := config.Data()["region"] + s3url := "" + s3Config := &aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + } + if region == "minio" { + s3url = config.Data()["s3Url"] + s3Config = &aws.Config{ + Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""), + Endpoint: aws.String(s3url), + Region: aws.String(region), + DisableSSL: aws.Bool(true), + S3ForcePathStyle: aws.Bool(true), + } + } + sess, err := session.NewSession(s3Config) + if err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + svc := s3.New(sess) + fullPrefix := strings.Trim(bslPrefix, "/") + "/" + strings.Trim(backupObject, "/") + "/" + iter := s3manager.NewDeleteListIterator(svc, &s3.ListObjectsInput{ + Bucket: aws.String(bslBucket), + Prefix: aws.String(fullPrefix), + }) + + if err := s3manager.NewBatchDeleteWithClient(svc).Delete(aws.BackgroundContext(), iter); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + fmt.Printf("Deleted object(s) from bucket: %s %s \n", bslBucket, fullPrefix) + return nil +} diff --git a/test/e2e/util/providers/azure_utils.go b/test/e2e/util/providers/azure_utils.go new file mode 100644 index 000000000..d9ee4c6ca --- /dev/null +++ b/test/e2e/util/providers/azure_utils.go @@ -0,0 +1,258 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "log" + "net/url" + "os" + "strings" + + "github.com/Azure/azure-pipeline-go/pipeline" + storagemgmt "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/Azure/go-autorest/autorest/azure/auth" + "github.com/joho/godotenv" + "github.com/pkg/errors" + "golang.org/x/net/context" + + "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" +) + +type AzureStorage string + +const ( + subscriptionIDEnvVar = "AZURE_SUBSCRIPTION_ID" + cloudNameEnvVar = "AZURE_CLOUD_NAME" + resourceGroupEnvVar = "AZURE_RESOURCE_GROUP" + storageAccountKey = "AZURE_STORAGE_ACCOUNT_ACCESS_KEY" + storageAccount = "storageAccount" + subscriptionID = "subscriptionId" + resourceGroup = "resourceGroup" +) + +func getStorageCredential(cloudCredentialsFile, bslConfig string) (string, string, error) { + config := flag.NewMap() + config.Set(bslConfig) + accountName := config.Data()[storageAccount] + // Account name must be provided in config + if len(accountName) == 0 { + return "", "", errors.New("Please provide bucket as Azure account name ") + } + subscriptionID := config.Data()[subscriptionID] + resourceGroupCfg := config.Data()[resourceGroup] + accountKey, err := getStorageAccountKey(cloudCredentialsFile, accountName, subscriptionID, resourceGroupCfg) + if err != nil { + return "", "", errors.Wrapf(err, "Fail to get storage key of bucket %s", accountName) + } + return accountName, accountKey, nil +} +func loadCredentialsIntoEnv(credentialsFile string) error { + if credentialsFile == "" { + return nil + } + + if err := godotenv.Overload(credentialsFile); err != nil { + return errors.Wrapf(err, "error loading environment from credentials file (%s)", credentialsFile) + } + return nil +} +func parseAzureEnvironment(cloudName string) (*azure.Environment, error) { + if cloudName == "" { + fmt.Println("cloudName is empty") + return &azure.PublicCloud, nil + } + + env, err := azure.EnvironmentFromName(cloudName) + return &env, errors.WithStack(err) +} +func getStorageAccountKey(credentialsFile, accountName, subscriptionID, resourceGroupCfg string) (string, error) { + if err := loadCredentialsIntoEnv(credentialsFile); err != nil { + return "", err + } + storageKey := os.Getenv(storageAccountKey) + if storageKey != "" { + return storageKey, nil + } + if os.Getenv(cloudNameEnvVar) == "" { + return "", errors.New("Credential file should contain AZURE_CLOUD_NAME") + } + var resourceGroup string + if os.Getenv(resourceGroupEnvVar) == "" { + if resourceGroupCfg == "" { + return "", errors.New("Credential file should contain AZURE_RESOURCE_GROUP or AZURE_STORAGE_ACCOUNT_ACCESS_KEY") + } else { + resourceGroup = resourceGroupCfg + } + } else { + resourceGroup = os.Getenv(resourceGroupEnvVar) + } + // get Azure cloud from AZURE_CLOUD_NAME, if it exists. If the env var does not + // exist, parseAzureEnvironment will return azure.PublicCloud. + env, err := parseAzureEnvironment(os.Getenv(cloudNameEnvVar)) + if err != nil { + return "", errors.Wrap(err, "unable to parse azure cloud name environment variable") + } + + // get subscription ID from object store config or AZURE_SUBSCRIPTION_ID environment variable + if subscriptionID == "" { + return "", errors.New("azure subscription ID not found in object store's config or in environment variable") + } + + authorizer, err := auth.NewAuthorizerFromEnvironment() + if err != nil { + return "", errors.Wrap(err, "error getting authorizer from environment") + } + + // get storageAccountsClient + storageAccountsClient := storagemgmt.NewAccountsClientWithBaseURI(env.ResourceManagerEndpoint, subscriptionID) + storageAccountsClient.Authorizer = authorizer + + // get storage key + res, err := storageAccountsClient.ListKeys(context.TODO(), resourceGroup, accountName, storagemgmt.Kerb) + if err != nil { + return "", errors.WithStack(err) + } + if res.Keys == nil || len(*res.Keys) == 0 { + return "", errors.New("No storage keys found") + } + + for _, key := range *res.Keys { + // uppercase both strings for comparison because the ListKeys call returns e.g. "FULL" but + // the storagemgmt.Full constant in the SDK is defined as "Full". + if strings.EqualFold(string(key.Permissions), string(storagemgmt.Full)) { + storageKey = *key.Value + break + } + } + + if storageKey == "" { + return "", errors.New("No storage key with Full permissions found") + } + + return storageKey, nil +} +func handleErrors(err error) { + if err != nil { + if serr, ok := err.(azblob.StorageError); ok { // This error is a Service-specific + switch serr.ServiceCode() { // Compare serviceCode to ServiceCodeXxx constants + case azblob.ServiceCodeContainerAlreadyExists: + return + } + } + log.Fatal(err) + } +} + +func deleteBlob(p pipeline.Pipeline, accountName, containerName, blobName string) error { + ctx := context.Background() + + URL_BLOB, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", accountName, containerName, blobName)) + if err != nil { + return errors.Wrapf(err, "Fail to url.Parse") + } + blobURL := azblob.NewBlobURL(*URL_BLOB, p) + _, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + return err +} +func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { + accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig) + if err != nil { + log.Fatal("Fail to get : accountName and accountKey, " + err.Error()) + } + + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + log.Fatal("Invalid credentials with error: " + err.Error()) + } + p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + + containerName := bslBucket + + URL, _ := url.Parse( + fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName)) + + containerURL := azblob.NewContainerURL(*URL, p) + + // Create the container, if container is already exist, then do nothing + ctx := context.Background() + _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) + handleErrors(err) + + fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupObject, containerName) + for marker := (azblob.Marker{}); marker.NotDone(); { + listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{}) + if err != nil { + return false, errors.Wrapf(err, "Fail to create gcloud client") + } + marker = listBlob.NextMarker + + for _, blobInfo := range listBlob.Segment.BlobItems { + if strings.Contains(blobInfo.Name, backupObject) { + fmt.Printf("Blob name: %s exist in %s\n", backupObject, blobInfo.Name) + return true, nil + } + } + } + return false, nil +} + +func (s AzureStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error { + ctx := context.Background() + accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig) + if err != nil { + return errors.Wrapf(err, "Fail to get storage account name and key of bucket %s", bslBucket) + } + + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + log.Fatal("Invalid credentials with error: " + err.Error()) + } + p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + + containerName := bslBucket + + URL, _ := url.Parse( + fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, containerName)) + + containerURL := azblob.NewContainerURL(*URL, p) + _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) + handleErrors(err) + + fmt.Println("Listing the blobs in the container:") + for marker := (azblob.Marker{}); marker.NotDone(); { + listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{}) + if err != nil { + return errors.Wrapf(err, "Fail to create gcloud client") + } + + marker = listBlob.NextMarker + for _, blobInfo := range listBlob.Segment.BlobItems { + + if strings.Contains(blobInfo.Name, bslPrefix+backupObject+"/") { + deleteBlob(p, accountName, containerName, blobInfo.Name) + if err != nil { + log.Fatal("Invalid credentials with error: " + err.Error()) + } + fmt.Printf("Deleted blob: %s according to backup resource %s\n", blobInfo.Name, bslPrefix+backupObject+"/") + } + } + } + return nil +} diff --git a/test/e2e/util/providers/common.go b/test/e2e/util/providers/common.go new file mode 100644 index 000000000..aa00065c9 --- /dev/null +++ b/test/e2e/util/providers/common.go @@ -0,0 +1,105 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "strings" + "time" + + "github.com/pkg/errors" +) + +type ObjectsInStorage interface { + IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) + DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error +} + +func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { + fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage %s", backupName, bslPrefix) + exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) + if !exist { + return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected", backupName)) + } + fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket) + return nil +} +func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error { + var err error + var exist bool + fmt.Printf("|| VERIFICATION || - Backup %s should not exist in storage %s", backupName, bslPrefix) + for i := 0; i < retryTimes; i++ { + exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) + if err != nil { + return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store", backupName) + } + if !exist { + fmt.Printf("|| EXPECTED || - Backup %s is not in object store\n", backupName) + return nil + } + time.Sleep(1 * time.Minute) + } + return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion", backupName)) +} +func getProvider(cloudProvider string) (ObjectsInStorage, error) { + var s ObjectsInStorage + switch cloudProvider { + case "aws", "vsphere": + aws := AWSStorage("") + s = &aws + case "gcp": + gcs := GCSStorage("") + s = &gcs + case "azure": + az := AzureStorage("") + s = &az + default: + return nil, errors.New(fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + } + return s, nil +} +func getFullPrefix(bslPrefix, subPrefix string) string { + if bslPrefix == "" { + bslPrefix = subPrefix + "/" + } else { + //subPrefix must have surfix "/", so that objects under it can be listed + bslPrefix = strings.Trim(bslPrefix, "/") + "/" + strings.Trim(subPrefix, "/") + "/" + } + return bslPrefix +} +func IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) (bool, error) { + bslPrefix = getFullPrefix(bslPrefix, subPrefix) + s, err := getProvider(cloudProvider) + if err != nil { + return false, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + } + return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) +} + +func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { + bslPrefix = getFullPrefix(bslPrefix, subPrefix) + fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s", backupName, bslPrefix) + s, err := getProvider(cloudProvider) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + } + err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("Fail to delete %s", bslPrefix)) + } + return nil +} diff --git a/test/e2e/util/providers/gcloud_utils.go b/test/e2e/util/providers/gcloud_utils.go new file mode 100644 index 000000000..fdc01b081 --- /dev/null +++ b/test/e2e/util/providers/gcloud_utils.go @@ -0,0 +1,99 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providers + +import ( + "fmt" + "strings" + + "cloud.google.com/go/storage" + "github.com/pkg/errors" + "golang.org/x/net/context" + "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +type GCSStorage string + +func (s GCSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { + q := &storage.Query{ + Prefix: bslPrefix, + } + + ctx := context.Background() + client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile)) + if err != nil { + return false, errors.Wrapf(err, "Fail to create gcloud client") + } + iter := client.Bucket(bslBucket).Objects(context.Background(), q) + for { + obj, err := iter.Next() + if err == iterator.Done { + return false, errors.Wrapf(err, fmt.Sprintf("Backup %s was not found under prefix %s \n", backupObject, bslPrefix)) + } + if err != nil { + return false, errors.WithStack(err) + } + if obj.Name == bslPrefix { + fmt.Println("Ignore GCS prefix itself") + continue + } + if strings.Contains(obj.Name, bslPrefix+backupObject+"/") { + fmt.Printf("Found delete-object %s of %s in bucket %s \n", backupObject, obj.Name, bslBucket) + return true, nil + } + } +} +func (s GCSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error { + q := &storage.Query{ + Prefix: bslPrefix, + } + + ctx := context.Background() + client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile)) + if err != nil { + return errors.Wrapf(err, "Fail to create gcloud client") + } + bucket := client.Bucket(bslBucket) + iter := bucket.Objects(context.Background(), q) + deleted := false + for { + obj, err := iter.Next() + if err == iterator.Done { + fmt.Println(err) + if !deleted { + return errors.New("|| UNEXPECTED ||Backup object is not exist and was not deleted in object store") + } + return nil + } + if err != nil { + return errors.WithStack(err) + } + if obj.Name == bslPrefix { + fmt.Println("Ignore GCS prefix itself") + continue + } + // Only delete folder named as backupObject under prefix + if strings.Contains(obj.Name, bslPrefix+backupObject+"/") { + if err = bucket.Object(obj.Name).Delete(ctx); err != nil { + return errors.Wrapf(err, fmt.Sprintf("Fail to delete object %s in bucket %s", obj.Name, bslBucket)) + } + fmt.Printf("Delete item: %s\n", obj.Name) + deleted = true + } + } +} diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index d32c9f577..a7db0808e 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -41,6 +41,10 @@ import ( veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) +const ( + BackupObjectsPrefix = "backups" +) + var pluginsMatrix = map[string]map[string][]string{ "v1.4": { "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, @@ -388,6 +392,9 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro // installs them in the current Velero installation, skipping over those that are already installed. func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error { plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins) + fmt.Printf("addPlugins cmd =%v\n", addPlugins) + fmt.Printf("provider cmd = %v\n", provider) + fmt.Printf("plugins cmd = %v\n", plugins) if err != nil { return errors.WithMessage(err, "Failed to get plugins") } @@ -396,6 +403,7 @@ func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa stderrBuf := new(bytes.Buffer) installPluginCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "plugin", "add", plugin) + fmt.Printf("installPluginCmd cmd =%v\n", installPluginCmd) installPluginCmd.Stdout = stdoutBuf installPluginCmd.Stderr = stderrBuf @@ -557,3 +565,34 @@ func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) { return tmpfile, nil } +func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName string) error { + args := []string{"backup", "delete", backupName, "--confirm"} + + cmd := exec.CommandContext(ctx, veleroCLI, args...) + fmt.Println("Delete backup Command:" + cmd.String()) + stdout, stderr, err := veleroexec.RunCommand(cmd) + if err != nil { + return errors.Wrapf(err, "Fail to get delete backup, stdout=%s, stderr=%s", stdout, stderr) + } + + output := strings.Replace(stdout, "\n", " ", -1) + fmt.Println("Backup delete command output:" + output) + + args = []string{"backup", "get", backupName} + + retryTimes := 5 + for i := 1; i < retryTimes+1; i++ { + cmd = exec.CommandContext(ctx, veleroCLI, args...) + fmt.Printf("Try %d times to delete backup %s \n", i, cmd.String()) + stdout, stderr, err = veleroexec.RunCommand(cmd) + if err != nil { + if strings.Contains(stderr, "not found") { + fmt.Printf("|| EXPECTED || - Backup %s was deleted successfully according to message %s\n", backupName, stderr) + return nil + } + return errors.Wrapf(err, "Fail to get delete backup, stdout=%s, stderr=%s", stdout, stderr) + } + time.Sleep(1 * time.Minute) + } + return nil +}