From b4be7eccb9031793ae152013b7c265f19f80e6fe Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Tue, 3 Mar 2020 15:57:15 -0500 Subject: [PATCH 01/26] Document locations for CSI support Signed-off-by: Nolan Brubaker --- pkg/controller/backup_controller.go | 8 ++++++++ pkg/persistence/object_store.go | 3 ++- pkg/persistence/object_store_layout.go | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index ef7a91dc3..d1b5a46e0 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -40,6 +40,7 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/discovery" + "github.com/vmware-tanzu/velero/pkg/features" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" velerov1informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1" velerov1listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1" @@ -74,6 +75,8 @@ type backupController struct { formatFlag logging.Format } +// TODO (nrb): Add clients for the VS/VSContent here. +// How about creating them in the server iff EnableCSI, and if they're nil in the backup controller, don't worry about it? func NewBackupController( backupInformer velerov1informers.BackupInformer, client velerov1client.BackupsGetter, @@ -599,6 +602,11 @@ func recordBackupMetrics(log logrus.FieldLogger, backup *velerov1api.Backup, bac } func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File, backupStore persistence.BackupStore, log logrus.FieldLogger) []error { + if features.IsEnabled("EnableCSI") { + // Get the list of VolumeSnapshots & VolumeSnapshotContents associated with the backup and serialize them as JSON + // Probably shouldn't fetch them here as there's no client, but they should be passed in somehow. + } + errs := []error{} backupJSON := new(bytes.Buffer) diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 8ddf066a6..562393e78 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -42,6 +42,7 @@ type BackupInfo struct { PodVolumeBackups, VolumeSnapshots, BackupResourceList io.Reader + // Add io.Readers here for VS and VSContents lists? } // BackupStore defines operations for creating, retrieving, and deleting @@ -51,7 +52,7 @@ type BackupStore interface { ListBackups() ([]string, error) - PutBackup(info BackupInfo) error + PutBackup(info BackupInfo) error // Extend this method for a VS/VSContent parameter, or create a new one for the beta period? GetBackupMetadata(name string) (*velerov1api.Backup, error) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) GetPodVolumeBackups(name string) ([]*velerov1api.PodVolumeBackup, error) diff --git a/pkg/persistence/object_store_layout.go b/pkg/persistence/object_store_layout.go index ebf75af19..d1423fb0e 100644 --- a/pkg/persistence/object_store_layout.go +++ b/pkg/persistence/object_store_layout.go @@ -99,3 +99,5 @@ func (l *ObjectStoreLayout) getRestoreLogKey(restore string) string { func (l *ObjectStoreLayout) getRestoreResultsKey(restore string) string { return path.Join(l.subdirs["restores"], restore, fmt.Sprintf("restore-%s-results.gz", restore)) } + +// TODO: Add keys for VS & VSContents json.gz files From 232e1d892700f35b27249e50b0ca760675794437 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Wed, 4 Mar 2020 14:02:47 -0500 Subject: [PATCH 02/26] Add clients for CSI snapshots Signed-off-by: Nolan Brubaker --- go.mod | 9 +++--- go.sum | 43 +++++++++++++++++++++++++---- pkg/cmd/server/server.go | 20 ++++++++++++++ pkg/controller/backup_controller.go | 18 +++++++++++- 4 files changed, 78 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index d1bd7ccc8..5518c4b3e 100644 --- a/go.mod +++ b/go.mod @@ -11,17 +11,16 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect github.com/aws/aws-sdk-go v1.13.12 github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 // indirect - github.com/evanphx/json-patch v4.2.0+incompatible + github.com/evanphx/json-patch v4.5.0+incompatible github.com/go-ini/ini v1.28.2 // indirect github.com/gobwas/glob v0.2.3 github.com/gofrs/uuid v3.2.0+incompatible github.com/golang/protobuf v1.3.2 - github.com/google/go-cmp v0.3.1 // indirect - github.com/googleapis/gnostic v0.1.0 // indirect github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect github.com/joho/godotenv v1.3.0 + github.com/kubernetes-csi/external-snapshotter/v2 v2.0.1 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.0.0 github.com/robfig/cron v0.0.0-20170309132418-df38d32658d8 @@ -31,8 +30,8 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 - golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 - google.golang.org/grpc v1.23.1 + golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 + google.golang.org/grpc v1.26.0 k8s.io/api v0.17.4 k8s.io/apiextensions-apiserver v0.17.4 k8s.io/apimachinery v0.17.4 diff --git a/go.sum b/go.sum index 9dbb06fbc..a0f218cf7 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,10 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -90,8 +92,12 @@ github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -172,6 +178,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -199,8 +207,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= -github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= +github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -225,6 +233,8 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= +github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= @@ -253,6 +263,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= +github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= +github.com/kubernetes-csi/external-snapshotter/v2 v2.0.1 h1:cRf1gQAzIXC6043qgLMfV3/LLddLmcqi5/c2bkuxaGI= +github.com/kubernetes-csi/external-snapshotter/v2 v2.0.1/go.mod h1:vUEcwbrEpsQ/rDgaO8WTe1gVIY/4CCj0S4Q+UuOq5wA= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -292,6 +306,8 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -311,6 +327,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -385,6 +403,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -426,6 +445,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -456,6 +477,8 @@ golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f h1:72l8qCJ1nGxMGH26QVBVIxKd/D34cfGt0OvrPtpemyY= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -511,12 +534,17 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51 h1:Ex1mq5jaJof+kRnYi3SlYJ8KKa9Ao3NHyIT5XJ1gF6U= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1 h1:PlscBL5CvF+v1mNR82G+i4kACGq2JQvKDnNq7LSS65o= +google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= @@ -546,20 +574,24 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= k8s.io/api v0.17.4 h1:HbwOhDapkguO8lTAE8OX3hdF2qp8GtpC9CW/MQATXXo= k8s.io/api v0.17.4/go.mod h1:5qxx6vjmwUVG2nHQTKGlLts8Tbok8PzHl4vHtVFuZCA= k8s.io/apiextensions-apiserver v0.17.4 h1:ZKFnw3cJrGZ/9s6y+DerTF4FL+dmK0a04A++7JkmMho= k8s.io/apiextensions-apiserver v0.17.4/go.mod h1:rCbbbaFS/s3Qau3/1HbPlHblrWpFivoaLYccCffvQGI= +k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.17.1-beta.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.4 h1:UzM+38cPUJnzqSQ+E1PY4YxMHIzQyCg29LOoGfo79Zw= k8s.io/apimachinery v0.17.4/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g= -k8s.io/apiserver v0.17.4 h1:bYc9LvDPEF9xAL3fhbDzqNOQOAnNF2ZYCrMW8v52/mE= k8s.io/apiserver v0.17.4/go.mod h1:5ZDQ6Xr5MNBxyi3iUZXS84QOhZl+W7Oq2us/29c0j9I= k8s.io/cli-runtime v0.17.4 h1:ZIJdxpBEszZqUhydrCoiI5rLXS2J/1AF5xFok2QJ9bc= k8s.io/cli-runtime v0.17.4/go.mod h1:IVW4zrKKx/8gBgNNkhiUIc7nZbVVNhc1+HcQh+PiNHc= +k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= k8s.io/client-go v0.17.4 h1:VVdVbpTY70jiNHS1eiFkUt7ZIJX3txd29nDxxXH4en8= k8s.io/client-go v0.17.4/go.mod h1:ouF6o5pz3is8qU0/qYL2RnoxOPqgfuidYLowytyLJmc= +k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= k8s.io/code-generator v0.17.4/go.mod h1:l8BLVwASXQZTo2xamW5mQNFCe1XPiAesVq7Y1t7PiQQ= -k8s.io/component-base v0.17.4 h1:H9cdWZyiGVJfWmWIcHd66IsNBWTk1iEgU7D4kJksEnw= +k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= k8s.io/component-base v0.17.4/go.mod h1:5BRqHMbbQPm2kKu35v3G+CpVq4K0RJKC7TRioF0I9lE= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -567,8 +599,8 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/kubernetes v1.14.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191218082557-f07c713de883 h1:TA8t8OLS8m3/0dtTckekO0pCQ7qMnD19fsZTQEgCSKQ= k8s.io/utils v0.0.0-20191218082557-f07c713de883/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= @@ -580,7 +612,6 @@ modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06 h1:zD2IemQ4LmOcAumeiyDWXKUI2SO0NYDe3H6QGvPOVgU= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 2bc9bf93a..0c518d8a7 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -43,6 +43,9 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + snapshotter "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" + api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/buildinfo" @@ -217,6 +220,7 @@ type server struct { kubeClientConfig *rest.Config kubeClient kubernetes.Interface veleroClient clientset.Interface + csiSnapClient snapshotter.Interface discoveryClient discovery.DiscoveryInterface discoveryHelper velerodiscovery.Helper dynamicClient dynamic.Interface @@ -588,6 +592,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.sharedInformerFactory.Velero().V1().Backups(), s.veleroClient.VeleroV1(), s.discoveryHelper, + s.csiSnapClient.SnapshotV1beta1(), backupper, s.logger, s.logLevel, @@ -832,3 +837,18 @@ func (s *server) runProfiler() { s.logger.WithError(errors.WithStack(err)).Error("error running profiler http server") } } + +// Helper methods that only return clients if the user has turned on CSI. +func volumeSnapshotClientIfCSIEnabled(i snapshotter.Interface) snapshotv1beta1.VolumeSnapshotInterface { + if features.IsEnabled("EnableCSI") { + return i.SnapshotV1beta1().VolumeSnapshots("") // is this correct? we need the namespace? Could we get this lazily later, via the backup? + } + return nil +} + +func voluemSnapshotContentClientIfCSIEnabled(i snapshotter.Interface) snapshotv1beta1.VolumeSnapshotContentInterface { + if features.IsEnabled("EnableCSI") { + return i.SnapshotV1beta1().VolumeSnapshotContents() + } + return nil +} diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index d1b5a46e0..7bb9be83e 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -37,6 +37,8 @@ import ( kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/cache" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/discovery" @@ -61,6 +63,7 @@ type backupController struct { backupper pkgbackup.Backupper lister velerov1listers.BackupLister client velerov1client.BackupsGetter + csiClient snapshotv1beta1.SnapshotV1beta1Interface clock clock.Clock backupLogLevel logrus.Level newPluginManager func(logrus.FieldLogger) clientmgmt.Manager @@ -75,12 +78,13 @@ type backupController struct { formatFlag logging.Format } -// TODO (nrb): Add clients for the VS/VSContent here. +// TODO(nrb-csi): Add clients for the VS/VSContent here. // How about creating them in the server iff EnableCSI, and if they're nil in the backup controller, don't worry about it? func NewBackupController( backupInformer velerov1informers.BackupInformer, client velerov1client.BackupsGetter, discoveryHelper discovery.Helper, + csiClient snapshotv1beta1.SnapshotV1beta1Interface, backupper pkgbackup.Backupper, logger logrus.FieldLogger, backupLogLevel logrus.Level, @@ -100,6 +104,7 @@ func NewBackupController( backupper: backupper, lister: backupInformer.Lister(), client: client, + csiClient: csiClient, clock: &clock.RealClock{}, backupLogLevel: backupLogLevel, newPluginManager: newPluginManager, @@ -571,6 +576,17 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { backup.Status.Phase = velerov1api.BackupPhaseCompleted } + // TODO(nrb-csi): Fetch VS(C)s here, based on includes/exludes? + if features.IsEnabled("EnableCSI") { + selector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + "velero.io/backup-name": backup.Name, + }, + } + // Need to iterate through included namespaces to get all VSs made by the backup + c.csiClient.VolumeSnapshots("").List(metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(&selector)}) + } + if errs := persistBackup(backup, backupFile, logFile, backupStore, c.logger); len(errs) > 0 { fatalErrs = append(fatalErrs, errs...) } From 6a7beaf5ce330d37c711a60761f63ef1b10a053a Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 12 Mar 2020 13:32:58 -0400 Subject: [PATCH 03/26] Plumb CSI listers through to backup controller Account for having CSI enabled or not, as well as having the snapshot CRDs installed in the kubernetes cluster. Signed-off-by: Nolan Brubaker --- pkg/cmd/server/server.go | 113 ++++++++++++++++++---------- pkg/controller/backup_controller.go | 94 ++++++++++++----------- 2 files changed, 122 insertions(+), 85 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 0c518d8a7..43699a955 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -43,8 +43,10 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - snapshotter "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" - snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" + snapshotv1beta1api "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + snapshotvebeta1client "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + snapshotvebeta1informers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions" + snapshotv1beta1listers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/backup" @@ -215,24 +217,25 @@ func NewCommand(f client.Factory) *cobra.Command { } type server struct { - namespace string - metricsAddress string - kubeClientConfig *rest.Config - kubeClient kubernetes.Interface - veleroClient clientset.Interface - csiSnapClient snapshotter.Interface - discoveryClient discovery.DiscoveryInterface - discoveryHelper velerodiscovery.Helper - dynamicClient dynamic.Interface - sharedInformerFactory informers.SharedInformerFactory - ctx context.Context - cancelFunc context.CancelFunc - logger logrus.FieldLogger - logLevel logrus.Level - pluginRegistry clientmgmt.Registry - resticManager restic.RepositoryManager - metrics *metrics.ServerMetrics - config serverConfig + namespace string + metricsAddress string + kubeClientConfig *rest.Config + kubeClient kubernetes.Interface + veleroClient clientset.Interface + csiSnapClient snapshotvebeta1client.Interface + discoveryClient discovery.DiscoveryInterface + discoveryHelper velerodiscovery.Helper + dynamicClient dynamic.Interface + sharedInformerFactory informers.SharedInformerFactory + snapshotterSharedInformerFactory snapshotvebeta1informers.SharedInformerFactory + ctx context.Context + cancelFunc context.CancelFunc + logger logrus.FieldLogger + logLevel logrus.Level + pluginRegistry clientmgmt.Registry + resticManager restic.RepositoryManager + metrics *metrics.ServerMetrics + config serverConfig } func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*server, error) { @@ -270,9 +273,19 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s clientConfig, err := f.ClientConfig() if err != nil { + cancelFunc() return nil, err } + var csiSnapClient *snapshotvebeta1client.Clientset + if features.IsEnabled("EnableCSI") { + csiSnapClient, err = snapshotvebeta1client.NewForConfig(clientConfig) + if err != nil { + cancelFunc() + return nil, err + } + } + s := &server{ namespace: f.Namespace(), metricsAddress: config.metricsAddress, @@ -282,12 +295,15 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s discoveryClient: veleroClient.Discovery(), dynamicClient: dynamicClient, sharedInformerFactory: informers.NewSharedInformerFactoryWithOptions(veleroClient, 0, informers.WithNamespace(f.Namespace())), - ctx: ctx, - cancelFunc: cancelFunc, - logger: logger, - logLevel: logger.Level, - pluginRegistry: pluginRegistry, - config: config, + // If no namespace is specified, all namespaces are watched. + // This is desirable for VolumeSnapshots, as we want to query for all VolumeSnapshots across all namespaces using this informer + snapshotterSharedInformerFactory: snapshotvebeta1informers.NewSharedInformerFactoryWithOptions(csiSnapClient, 0), + ctx: ctx, + cancelFunc: cancelFunc, + logger: logger, + logLevel: logger.Level, + pluginRegistry: pluginRegistry, + config: config, } return s, nil @@ -588,6 +604,27 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string ) cmd.CheckError(err) + // Make empty listers that will only be populated if CSI is properly enabled. + var vsLister snapshotv1beta1listers.VolumeSnapshotLister + var vscLister snapshotv1beta1listers.VolumeSnapshotContentLister + + // We don't care about the returned types, just whether or not there's a 404 error. + _, err = s.discoveryClient.ServerResourcesForGroupVersion(snapshotv1beta1api.SchemeGroupVersion.String()) + switch { + case apierrors.IsNotFound(err) && !features.IsEnabled("EnableCSI"): + // Normal operating mode - CSI isn't enabled, so don't actually error + case apierrors.IsNotFound(err) && features.IsEnabled("EnableCSI"): + // CSI is enabled, but the required CRDs aren't installed, so halt. + s.logger.Fatalf("The 'EnableCSI' feature flag was specified, but CSI API group [%s] was not found.", snapshotv1beta1api.SchemeGroupVersion.String()) + case err == nil && features.IsEnabled("EnableCSI"): + // CSI is enabled, and the resources were found. + // Instantiate the listers fully + vsLister = s.snapshotterSharedInformerFactory.Snapshot().V1beta1().VolumeSnapshots().Lister() + vscLister = s.snapshotterSharedInformerFactory.Snapshot().V1beta1().VolumeSnapshotContents().Lister() + case err != nil: + cmd.CheckError(err) + } + backupController := controller.NewBackupController( s.sharedInformerFactory.Velero().V1().Backups(), s.veleroClient.VeleroV1(), @@ -605,6 +642,8 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string defaultVolumeSnapshotLocations, s.metrics, s.config.formatFlag.Parse(), + vsLister, + vscLister, ) return controllerRunInfo{ @@ -793,10 +832,17 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string // start the informers & and wait for the caches to sync s.sharedInformerFactory.Start(ctx.Done()) + s.snapshotterSharedInformerFactory.Start(ctx.Done()) s.logger.Info("Waiting for informer caches to sync") cacheSyncResults := s.sharedInformerFactory.WaitForCacheSync(ctx.Done()) + csiCacheSyncResults := s.snapshotterSharedInformerFactory.WaitForCacheSync(ctx.Done()) s.logger.Info("Done waiting for informer caches to sync") + // Append our CSI informer types into the larger list of caches, so we can check them all at once + for informer, synced := range csiCacheSyncResults { + cacheSyncResults[informer] = synced + } + for informer, synced := range cacheSyncResults { if !synced { return errors.Errorf("cache was not synced for informer %v", informer) @@ -837,18 +883,3 @@ func (s *server) runProfiler() { s.logger.WithError(errors.WithStack(err)).Error("error running profiler http server") } } - -// Helper methods that only return clients if the user has turned on CSI. -func volumeSnapshotClientIfCSIEnabled(i snapshotter.Interface) snapshotv1beta1.VolumeSnapshotInterface { - if features.IsEnabled("EnableCSI") { - return i.SnapshotV1beta1().VolumeSnapshots("") // is this correct? we need the namespace? Could we get this lazily later, via the backup? - } - return nil -} - -func voluemSnapshotContentClientIfCSIEnabled(i snapshotter.Interface) snapshotv1beta1.VolumeSnapshotContentInterface { - if features.IsEnabled("EnableCSI") { - return i.SnapshotV1beta1().VolumeSnapshotContents() - } - return nil -} diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 7bb9be83e..705edd5d2 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -38,7 +38,9 @@ import ( "k8s.io/client-go/tools/cache" snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" + snapshotv1beta1listers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" + v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/discovery" @@ -59,23 +61,25 @@ import ( type backupController struct { *genericController - discoveryHelper discovery.Helper - backupper pkgbackup.Backupper - lister velerov1listers.BackupLister - client velerov1client.BackupsGetter - csiClient snapshotv1beta1.SnapshotV1beta1Interface - clock clock.Clock - backupLogLevel logrus.Level - newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - backupTracker BackupTracker - backupLocationLister velerov1listers.BackupStorageLocationLister - defaultBackupLocation string - defaultBackupTTL time.Duration - snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister - defaultSnapshotLocations map[string]string - metrics *metrics.ServerMetrics - newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) - formatFlag logging.Format + discoveryHelper discovery.Helper + backupper pkgbackup.Backupper + lister velerov1listers.BackupLister + client velerov1client.BackupsGetter + csiClient snapshotv1beta1.SnapshotV1beta1Interface + clock clock.Clock + backupLogLevel logrus.Level + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager + backupTracker BackupTracker + backupLocationLister velerov1listers.BackupStorageLocationLister + defaultBackupLocation string + defaultBackupTTL time.Duration + snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister + defaultSnapshotLocations map[string]string + metrics *metrics.ServerMetrics + newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) + formatFlag logging.Format + volumeSnapshotLister snapshotv1beta1listers.VolumeSnapshotLister + volumeSnapshotContentLister snapshotv1beta1listers.VolumeSnapshotContentLister } // TODO(nrb-csi): Add clients for the VS/VSContent here. @@ -97,27 +101,30 @@ func NewBackupController( defaultSnapshotLocations map[string]string, metrics *metrics.ServerMetrics, formatFlag logging.Format, + volumeSnapshotLister snapshotv1beta1listers.VolumeSnapshotLister, + volumeSnapshotContentLister snapshotv1beta1listers.VolumeSnapshotContentLister, ) Interface { c := &backupController{ - genericController: newGenericController("backup", logger), - discoveryHelper: discoveryHelper, - backupper: backupper, - lister: backupInformer.Lister(), - client: client, - csiClient: csiClient, - clock: &clock.RealClock{}, - backupLogLevel: backupLogLevel, - newPluginManager: newPluginManager, - backupTracker: backupTracker, - backupLocationLister: backupLocationLister, - defaultBackupLocation: defaultBackupLocation, - defaultBackupTTL: defaultBackupTTL, - snapshotLocationLister: volumeSnapshotLocationLister, - defaultSnapshotLocations: defaultSnapshotLocations, - metrics: metrics, - formatFlag: formatFlag, - - newBackupStore: persistence.NewObjectBackupStore, + genericController: newGenericController("backup", logger), + discoveryHelper: discoveryHelper, + backupper: backupper, + lister: backupInformer.Lister(), + client: client, + csiClient: csiClient, + clock: &clock.RealClock{}, + backupLogLevel: backupLogLevel, + newPluginManager: newPluginManager, + backupTracker: backupTracker, + backupLocationLister: backupLocationLister, + defaultBackupLocation: defaultBackupLocation, + defaultBackupTTL: defaultBackupTTL, + snapshotLocationLister: volumeSnapshotLocationLister, + defaultSnapshotLocations: defaultSnapshotLocations, + metrics: metrics, + formatFlag: formatFlag, + volumeSnapshotLister: volumeSnapshotLister, + volumeSnapshotContentLister: volumeSnapshotContentLister, + newBackupStore: persistence.NewObjectBackupStore, } c.syncHandler = c.processBackup @@ -576,17 +583,16 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { backup.Status.Phase = velerov1api.BackupPhaseCompleted } - // TODO(nrb-csi): Fetch VS(C)s here, based on includes/exludes? + // TODO(nrb-csi): Only run listers methods if the listers aren't nil, just in case if features.IsEnabled("EnableCSI") { - selector := metav1.LabelSelector{ - MatchLabels: map[string]string{ - "velero.io/backup-name": backup.Name, - }, - } - // Need to iterate through included namespaces to get all VSs made by the backup - c.csiClient.VolumeSnapshots("").List(metav1.ListOptions{LabelSelector: metav1.FormatLabelSelector(&selector)}) + selector := labels.SelectorFromSet(map[string]string{v1.BackupNameLabel: backup.Name}) + + // TODO: (nrb-csi): assign return values, handle errors + c.volumeSnapshotLister.List(selector) + c.volumeSnapshotContentLister.List(selector) } + // TODO(nrb-csi): pass VS(C) in to persistBackup here if errs := persistBackup(backup, backupFile, logFile, backupStore, c.logger); len(errs) > 0 { fatalErrs = append(fatalErrs, errs...) } From aff529e5d5a17adf210469914e3a4d7af7b2c440 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 12 Mar 2020 17:01:14 -0400 Subject: [PATCH 04/26] Upload CSI volumesnapshots associated with backup Signed-off-by: Nolan Brubaker --- pkg/controller/backup_controller.go | 94 ++++++++++++++++++-------- pkg/persistence/object_store.go | 33 ++++++++- pkg/persistence/object_store_layout.go | 7 ++ 3 files changed, 104 insertions(+), 30 deletions(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 705edd5d2..5747e3ea7 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -37,6 +37,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/cache" + snapshotv1beta1api "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" snapshotv1beta1listers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" @@ -550,6 +551,27 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { fatalErrs = append(fatalErrs, err) } + // Empty slices here so that they can be passed in to the persistBackup call later, regardless of whether or not CSI's enabled. + // This way, we only make the Lister call if the feature flag's on. + var volumeSnapshots []*snapshotv1beta1api.VolumeSnapshot + var volumeSnapshotContents []*snapshotv1beta1api.VolumeSnapshotContent + if features.IsEnabled("EnableCSI") { + selector := labels.SelectorFromSet(map[string]string{v1.BackupNameLabel: backup.Name}) + + // TODO(nrb-csi): Only run listers methods if the listers aren't nil, just in case + volumeSnapshots, err = c.volumeSnapshotLister.List(selector) + if err != nil { + //TODO: Is this right? + backupLog.Error(err) + } + volumeSnapshotContents, err = c.volumeSnapshotContentLister.List(selector) + if err != nil { + //TODO: Is this right? + backupLog.Error(err) + + } + } + // Mark completion timestamp before serializing and uploading. // Otherwise, the JSON file in object storage has a CompletionTimestamp of 'null'. backup.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} @@ -561,6 +583,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { } } + // TODO(nrb-csi): How do we handle backup metrics with CSI snapshots? Log an issue for this recordBackupMetrics(backupLog, backup.Backup, backupFile, c.metrics) if err := gzippedLogFile.Close(); err != nil { @@ -583,17 +606,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { backup.Status.Phase = velerov1api.BackupPhaseCompleted } - // TODO(nrb-csi): Only run listers methods if the listers aren't nil, just in case - if features.IsEnabled("EnableCSI") { - selector := labels.SelectorFromSet(map[string]string{v1.BackupNameLabel: backup.Name}) - - // TODO: (nrb-csi): assign return values, handle errors - c.volumeSnapshotLister.List(selector) - c.volumeSnapshotContentLister.List(selector) - } - - // TODO(nrb-csi): pass VS(C) in to persistBackup here - if errs := persistBackup(backup, backupFile, logFile, backupStore, c.logger); len(errs) > 0 { + if errs := persistBackup(backup, backupFile, logFile, backupStore, c.logger, volumeSnapshots, volumeSnapshotContents); len(errs) > 0 { fatalErrs = append(fatalErrs, errs...) } @@ -623,11 +636,13 @@ func recordBackupMetrics(log logrus.FieldLogger, backup *velerov1api.Backup, bac serverMetrics.RegisterVolumeSnapshotFailures(backupScheduleName, backup.Status.VolumeSnapshotsAttempted-backup.Status.VolumeSnapshotsCompleted) } -func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File, backupStore persistence.BackupStore, log logrus.FieldLogger) []error { - if features.IsEnabled("EnableCSI") { - // Get the list of VolumeSnapshots & VolumeSnapshotContents associated with the backup and serialize them as JSON - // Probably shouldn't fetch them here as there's no client, but they should be passed in somehow. - } +func persistBackup(backup *pkgbackup.Request, + backupContents, backupLog *os.File, + backupStore persistence.BackupStore, + log logrus.FieldLogger, + csiVolumeSnapshots []*snapshotv1beta1api.VolumeSnapshot, + volumeSnapshotContents []*snapshotv1beta1api.VolumeSnapshotContent, +) []error { errs := []error{} backupJSON := new(bytes.Buffer) @@ -636,11 +651,12 @@ func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File errs = append(errs, errors.Wrap(err, "error encoding backup")) } - volumeSnapshots := new(bytes.Buffer) - gzw := gzip.NewWriter(volumeSnapshots) + // Velero-native volume snapshots + nativeVolumeSnapshots := new(bytes.Buffer) + gzw := gzip.NewWriter(nativeVolumeSnapshots) if err := json.NewEncoder(gzw).Encode(backup.VolumeSnapshots); err != nil { - errs = append(errs, errors.Wrap(err, "error encoding list of volume snapshots")) + errs = append(errs, errors.Wrap(err, "error encoding list of velero volume snapshots")) } if err := gzw.Close(); err != nil { errs = append(errs, errors.Wrap(err, "error closing gzip writer")) @@ -656,6 +672,26 @@ func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File errs = append(errs, errors.Wrap(err, "error closing gzip writer")) } + csiSnapshotJSON := new(bytes.Buffer) + gzw = gzip.NewWriter(csiSnapshotJSON) + + if err := json.NewEncoder(gzw).Encode(csiVolumeSnapshots); err != nil { + errs = append(errs, errors.Wrap(err, "error encoding list of csi volume snapshots")) + } + if err := gzw.Close(); err != nil { + errs = append(errs, errors.Wrap(err, "error closing gzip writer")) + } + + snapshotContentsJSON := new(bytes.Buffer) + gzw = gzip.NewWriter(snapshotContentsJSON) + + if err := json.NewEncoder(gzw).Encode(volumeSnapshotContents); err != nil { + errs = append(errs, errors.Wrap(err, "error encoding list of csi volume snapshots")) + } + if err := gzw.Close(); err != nil { + errs = append(errs, errors.Wrap(err, "error closing gzip writer")) + } + backupResourceList := new(bytes.Buffer) gzw = gzip.NewWriter(backupResourceList) @@ -670,18 +706,22 @@ func persistBackup(backup *pkgbackup.Request, backupContents, backupLog *os.File // Don't upload the JSON files or backup tarball if encoding to json fails. backupJSON = nil backupContents = nil - volumeSnapshots = nil + nativeVolumeSnapshots = nil backupResourceList = nil + csiSnapshotJSON = nil + snapshotContentsJSON = nil } backupInfo := persistence.BackupInfo{ - Name: backup.Name, - Metadata: backupJSON, - Contents: backupContents, - Log: backupLog, - PodVolumeBackups: podVolumeBackups, - VolumeSnapshots: volumeSnapshots, - BackupResourceList: backupResourceList, + Name: backup.Name, + Metadata: backupJSON, + Contents: backupContents, + Log: backupLog, + PodVolumeBackups: podVolumeBackups, + VolumeSnapshots: nativeVolumeSnapshots, + BackupResourceList: backupResourceList, + CSIVolumeSnapshots: csiSnapshotJSON, + VolumeSnapshotContents: snapshotContentsJSON, } if err := backupStore.PutBackup(backupInfo); err != nil { errs = append(errs, err) diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 562393e78..97c341e10 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -41,8 +41,9 @@ type BackupInfo struct { Log, PodVolumeBackups, VolumeSnapshots, - BackupResourceList io.Reader - // Add io.Readers here for VS and VSContents lists? + BackupResourceList, + CSIVolumeSnapshots, + VolumeSnapshotContents io.Reader } // BackupStore defines operations for creating, retrieving, and deleting @@ -52,11 +53,12 @@ type BackupStore interface { ListBackups() ([]string, error) - PutBackup(info BackupInfo) error // Extend this method for a VS/VSContent parameter, or create a new one for the beta period? + PutBackup(info BackupInfo) error GetBackupMetadata(name string) (*velerov1api.Backup, error) GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) GetPodVolumeBackups(name string) ([]*velerov1api.PodVolumeBackup, error) GetBackupContents(name string) (io.ReadCloser, error) + // TODO(nrb-csi): Do we need Get methods for the CSI types? // BackupExists checks if the backup metadata file exists in object storage. BackupExists(bucket, backupName string) (bool, error) @@ -252,6 +254,31 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error { return kerrors.NewAggregate(errs) } + // TODO(nrb-csi): only upload these if they're non-nil and/or if EnableCSI is on + // TODO(nrb-csi): Add tests? + if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getCSIVolumeSnapshotContentsKey(info.Name), info.CSIVolumeSnapshots); err != nil { + errs := []error{err} + + deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getCSIVolumeSnapshotContentsKey(info.Name)) + errs = append(errs, deleteErr) + + deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) + errs = append(errs, deleteErr) + + return kerrors.NewAggregate(errs) + } + + if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getVolumeSnapshotContentsKey(info.Name), info.VolumeSnapshotContents); err != nil { + errs := []error{err} + + deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getVolumeSnapshotContentsKey(info.Name)) + errs = append(errs, deleteErr) + + deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) + errs = append(errs, deleteErr) + return kerrors.NewAggregate(errs) + } + return nil } diff --git a/pkg/persistence/object_store_layout.go b/pkg/persistence/object_store_layout.go index d1423fb0e..baf2fa196 100644 --- a/pkg/persistence/object_store_layout.go +++ b/pkg/persistence/object_store_layout.go @@ -101,3 +101,10 @@ func (l *ObjectStoreLayout) getRestoreResultsKey(restore string) string { } // TODO: Add keys for VS & VSContents json.gz files +func (l *ObjectStoreLayout) getCSIVolumeSnapshotContentsKey(backup string) string { + return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csivolumesnapshotcontents.json.gz", backup)) +} + +func (l *ObjectStoreLayout) getVolumeSnapshotContentsKey(backup string) string { + return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-volumesnapshotcontents.json.gz", backup)) +} From 5de15c450b7c990b07b748c3930c31563e3671f4 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Mon, 30 Mar 2020 14:12:37 -0400 Subject: [PATCH 05/26] Address some review feedback on server Signed-off-by: Nolan Brubaker --- pkg/cmd/server/server.go | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 43699a955..ac9ca0dfd 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -270,10 +270,10 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s } ctx, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() clientConfig, err := f.ClientConfig() if err != nil { - cancelFunc() return nil, err } @@ -281,7 +281,6 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s if features.IsEnabled("EnableCSI") { csiSnapClient, err = snapshotvebeta1client.NewForConfig(clientConfig) if err != nil { - cancelFunc() return nil, err } } @@ -608,21 +607,23 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string var vsLister snapshotv1beta1listers.VolumeSnapshotLister var vscLister snapshotv1beta1listers.VolumeSnapshotContentLister - // We don't care about the returned types, just whether or not there's a 404 error. - _, err = s.discoveryClient.ServerResourcesForGroupVersion(snapshotv1beta1api.SchemeGroupVersion.String()) - switch { - case apierrors.IsNotFound(err) && !features.IsEnabled("EnableCSI"): - // Normal operating mode - CSI isn't enabled, so don't actually error - case apierrors.IsNotFound(err) && features.IsEnabled("EnableCSI"): - // CSI is enabled, but the required CRDs aren't installed, so halt. - s.logger.Fatalf("The 'EnableCSI' feature flag was specified, but CSI API group [%s] was not found.", snapshotv1beta1api.SchemeGroupVersion.String()) - case err == nil && features.IsEnabled("EnableCSI"): - // CSI is enabled, and the resources were found. - // Instantiate the listers fully - vsLister = s.snapshotterSharedInformerFactory.Snapshot().V1beta1().VolumeSnapshots().Lister() - vscLister = s.snapshotterSharedInformerFactory.Snapshot().V1beta1().VolumeSnapshotContents().Lister() - case err != nil: - cmd.CheckError(err) + // If CSI is enabled, check for the CSI groups and generate the listers + // If CSI isn't enabled, proceed normally. + if features.IsEnabled("EnableCSI") { + _, err = s.discoveryClient.ServerResourcesForGroupVersion(snapshotv1beta1api.SchemeGroupVersion.String()) + switch { + case apierrors.IsNotFound(err): + // CSI is enabled, but the required CRDs aren't installed, so halt. + s.logger.Fatalf("The 'EnableCSI' feature flag was specified, but CSI API group [%s] was not found.", snapshotv1beta1api.SchemeGroupVersion.String()) + case err == nil: + // CSI is enabled, and the resources were found. + // Instantiate the listers fully + s.logger.Debug("Creating CSI listers") + vsLister = s.snapshotterSharedInformerFactory.Snapshot().V1beta1().VolumeSnapshots().Lister() + vscLister = s.snapshotterSharedInformerFactory.Snapshot().V1beta1().VolumeSnapshotContents().Lister() + case err != nil: + cmd.CheckError(err) + } } backupController := controller.NewBackupController( From 5bde12939e469de10801a55060f13b37045a984d Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Mon, 30 Mar 2020 14:26:23 -0400 Subject: [PATCH 06/26] Address review feedback on object store Signed-off-by: Nolan Brubaker --- pkg/persistence/object_store.go | 36 ++++++++++++++------------ pkg/persistence/object_store_layout.go | 5 ++-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 97c341e10..2b4169553 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -58,7 +58,7 @@ type BackupStore interface { GetBackupVolumeSnapshots(name string) ([]*volume.Snapshot, error) GetPodVolumeBackups(name string) ([]*velerov1api.PodVolumeBackup, error) GetBackupContents(name string) (io.ReadCloser, error) - // TODO(nrb-csi): Do we need Get methods for the CSI types? + // TODO(nrb-csi): Any Get methods relevant to the CSI VolumeSnapshots should be added with the client-side PRs. // BackupExists checks if the backup metadata file exists in object storage. BackupExists(bucket, backupName string) (bool, error) @@ -218,6 +218,7 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error { return kerrors.NewAggregate([]error{err, deleteErr}) } + // TODO: How do we reduce duplication from here to the end of the function? if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getPodVolumeBackupsKey(info.Name), info.PodVolumeBackups); err != nil { errs := []error{err} @@ -254,29 +255,32 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error { return kerrors.NewAggregate(errs) } - // TODO(nrb-csi): only upload these if they're non-nil and/or if EnableCSI is on // TODO(nrb-csi): Add tests? - if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getCSIVolumeSnapshotContentsKey(info.Name), info.CSIVolumeSnapshots); err != nil { - errs := []error{err} + if info.CSIVolumeSnapshots != nil { + if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getCSIVolumeSnapshotKey(info.Name), info.CSIVolumeSnapshots); err != nil { + errs := []error{err} - deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getCSIVolumeSnapshotContentsKey(info.Name)) - errs = append(errs, deleteErr) + deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getCSIVolumeSnapshotKey(info.Name)) + errs = append(errs, deleteErr) - deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) - errs = append(errs, deleteErr) + deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) + errs = append(errs, deleteErr) - return kerrors.NewAggregate(errs) + return kerrors.NewAggregate(errs) + } } - if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getVolumeSnapshotContentsKey(info.Name), info.VolumeSnapshotContents); err != nil { - errs := []error{err} + if info.VolumeSnapshotContents != nil { + if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getVolumeSnapshotContentsKey(info.Name), info.VolumeSnapshotContents); err != nil { + errs := []error{err} - deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getVolumeSnapshotContentsKey(info.Name)) - errs = append(errs, deleteErr) + deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getVolumeSnapshotContentsKey(info.Name)) + errs = append(errs, deleteErr) - deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) - errs = append(errs, deleteErr) - return kerrors.NewAggregate(errs) + deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) + errs = append(errs, deleteErr) + return kerrors.NewAggregate(errs) + } } return nil diff --git a/pkg/persistence/object_store_layout.go b/pkg/persistence/object_store_layout.go index baf2fa196..70e955968 100644 --- a/pkg/persistence/object_store_layout.go +++ b/pkg/persistence/object_store_layout.go @@ -100,9 +100,8 @@ func (l *ObjectStoreLayout) getRestoreResultsKey(restore string) string { return path.Join(l.subdirs["restores"], restore, fmt.Sprintf("restore-%s-results.gz", restore)) } -// TODO: Add keys for VS & VSContents json.gz files -func (l *ObjectStoreLayout) getCSIVolumeSnapshotContentsKey(backup string) string { - return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csivolumesnapshotcontents.json.gz", backup)) +func (l *ObjectStoreLayout) getCSIVolumeSnapshotKey(backup string) string { + return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csivolumesnapshot.json.gz", backup)) } func (l *ObjectStoreLayout) getVolumeSnapshotContentsKey(backup string) string { From c71318bf1934be98c180cfcbbe64cc25e115aba0 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Mon, 30 Mar 2020 14:32:07 -0400 Subject: [PATCH 07/26] Address more review feedback Signed-off-by: Nolan Brubaker --- go.mod | 3 ++- go.sum | 10 ++++++---- pkg/controller/backup_controller.go | 5 ++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 5518c4b3e..a90d0de93 100644 --- a/go.mod +++ b/go.mod @@ -16,11 +16,12 @@ require ( github.com/gobwas/glob v0.2.3 github.com/gofrs/uuid v3.2.0+incompatible github.com/golang/protobuf v1.3.2 + github.com/google/go-cmp v0.3.1 // indirect github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect github.com/joho/godotenv v1.3.0 - github.com/kubernetes-csi/external-snapshotter/v2 v2.0.1 + github.com/kubernetes-csi/external-snapshotter/v2 v2.1.0 github.com/pkg/errors v0.8.1 github.com/prometheus/client_golang v1.0.0 github.com/robfig/cron v0.0.0-20170309132418-df38d32658d8 diff --git a/go.sum b/go.sum index a0f218cf7..37f99a9b1 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -178,7 +179,6 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -207,6 +207,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= @@ -265,8 +267,9 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= -github.com/kubernetes-csi/external-snapshotter/v2 v2.0.1 h1:cRf1gQAzIXC6043qgLMfV3/LLddLmcqi5/c2bkuxaGI= -github.com/kubernetes-csi/external-snapshotter/v2 v2.0.1/go.mod h1:vUEcwbrEpsQ/rDgaO8WTe1gVIY/4CCj0S4Q+UuOq5wA= +github.com/kubernetes-csi/external-snapshotter v1.2.2 h1:OPXoJydNqkWjhLwJ20dSqOhkmUYcpm+CCO0pYm+C8Q8= +github.com/kubernetes-csi/external-snapshotter/v2 v2.1.0 h1:a1cpbNAdOTHO7Lk5UO5tjcbYPEEamIxmzATt+pKoDhc= +github.com/kubernetes-csi/external-snapshotter/v2 v2.1.0/go.mod h1:dV5oB3U62KBdlf9ADWkMmjGd3USauqQtwIm2OZb5mqI= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -306,7 +309,6 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 5747e3ea7..ca7e0aaf7 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -41,7 +41,6 @@ import ( snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" snapshotv1beta1listers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" - v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/discovery" @@ -556,7 +555,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { var volumeSnapshots []*snapshotv1beta1api.VolumeSnapshot var volumeSnapshotContents []*snapshotv1beta1api.VolumeSnapshotContent if features.IsEnabled("EnableCSI") { - selector := labels.SelectorFromSet(map[string]string{v1.BackupNameLabel: backup.Name}) + selector := labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: backup.Name}) // TODO(nrb-csi): Only run listers methods if the listers aren't nil, just in case volumeSnapshots, err = c.volumeSnapshotLister.List(selector) @@ -643,7 +642,7 @@ func persistBackup(backup *pkgbackup.Request, csiVolumeSnapshots []*snapshotv1beta1api.VolumeSnapshot, volumeSnapshotContents []*snapshotv1beta1api.VolumeSnapshotContent, ) []error { - + // TODO(nrb-csi): Reduce duplication in this function errs := []error{} backupJSON := new(bytes.Buffer) From 6827a41cf7115b7229444ad779746521bba0d9bc Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Mon, 30 Mar 2020 14:33:34 -0400 Subject: [PATCH 08/26] Add dependency to external-snapshotter v2.0.1 All changes to go.mod/go.sum besides the external-snapshotter repo are a result of its transitive dependencies. Signed-off-by: Nolan Brubaker --- go.sum | 61 ---------------------------------------------------------- 1 file changed, 61 deletions(-) diff --git a/go.sum b/go.sum index 37f99a9b1..397a13641 100644 --- a/go.sum +++ b/go.sum @@ -67,10 +67,8 @@ github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHo github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1DoGKxLx3qTSI8Ttl6Xrqp6mw= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -88,20 +86,15 @@ github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 h1:llBx5m8Gk0lrA github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -116,56 +109,45 @@ github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70t github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5 h1:8b2ZgKfKIUTVQpTb77MoRDIMEIwvDVw40o3aOXdfYzI= github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.4 h1:5I4CCSqoWzT+82bBkNIvmLc0UOsoKKQ4Fz+3VxOB7SY= github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4 h1:csnOgcgAiuGoM/Po7PEpKDoNulCcF3FGbSnbHfxgjMI= github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= -github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.3 h1:eRfyY5SkaNJCAwmmMcADjY31ow9+N7MCLW7oRkbsINA= github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.5 h1:QhCBKRYqZR+SKo4gl1lPhPahope8/RLt6EVgY8X80w0= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= @@ -175,9 +157,7 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -202,23 +182,18 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= -github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd h1:rNuUHR+CvK1IS89MMtcF0EpcVMZtjKfPRp4MEmt/aTs= @@ -231,13 +206,10 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -250,24 +222,19 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= -github.com/kubernetes-csi/external-snapshotter v1.2.2 h1:OPXoJydNqkWjhLwJ20dSqOhkmUYcpm+CCO0pYm+C8Q8= github.com/kubernetes-csi/external-snapshotter/v2 v2.1.0 h1:a1cpbNAdOTHO7Lk5UO5tjcbYPEEamIxmzATt+pKoDhc= github.com/kubernetes-csi/external-snapshotter/v2 v2.1.0/go.mod h1:dV5oB3U62KBdlf9ADWkMmjGd3USauqQtwIm2OZb5mqI= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= @@ -278,7 +245,6 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -288,7 +254,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 h1:7GoSOOW2jpsfkntVKaS2rAr1TJqfcxotyaUcuxoZSzg= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -298,7 +263,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= @@ -307,11 +271,9 @@ github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -320,14 +282,12 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -346,13 +306,10 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -367,12 +324,10 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -382,19 +337,14 @@ github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2 h1:jxcFYjlkl8xaERsgLo+RNquI0epW6zuy/ZRQs6jnrFA= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -445,7 +395,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -477,7 +426,6 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220220014-0732a990476f h1:72l8qCJ1nGxMGH26QVBVIxKd/D34cfGt0OvrPtpemyY= golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -512,7 +460,6 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72 h1:bw9doJza/SFBEweII/rHQh338oozWyiFsBRHtrflcws= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= @@ -525,7 +472,6 @@ google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -534,7 +480,6 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51 h1:Ex1mq5jaJof+kRnYi3SlYJ8KKa9Ao3NHyIT5XJ1gF6U= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1 h1:PlscBL5CvF+v1mNR82G+i4kACGq2JQvKDnNq7LSS65o= google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -543,30 +488,24 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From b7142465fc86ff596641d536e5a5fb4601b8fc77 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Mon, 30 Mar 2020 15:04:41 -0400 Subject: [PATCH 09/26] Wrap usage of the CSI informer in flag checks Signed-off-by: Nolan Brubaker --- pkg/cmd/server/server.go | 59 ++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index ac9ca0dfd..c91f9f6bf 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -23,6 +23,7 @@ import ( "net/http" "net/http/pprof" "os" + "reflect" "strings" "sync" "time" @@ -227,7 +228,7 @@ type server struct { discoveryHelper velerodiscovery.Helper dynamicClient dynamic.Interface sharedInformerFactory informers.SharedInformerFactory - snapshotterSharedInformerFactory snapshotvebeta1informers.SharedInformerFactory + snapshotterSharedInformerFactory *CSIInformerFactoryWrapper ctx context.Context cancelFunc context.CancelFunc logger logrus.FieldLogger @@ -286,17 +287,15 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s } s := &server{ - namespace: f.Namespace(), - metricsAddress: config.metricsAddress, - kubeClientConfig: clientConfig, - kubeClient: kubeClient, - veleroClient: veleroClient, - discoveryClient: veleroClient.Discovery(), - dynamicClient: dynamicClient, - sharedInformerFactory: informers.NewSharedInformerFactoryWithOptions(veleroClient, 0, informers.WithNamespace(f.Namespace())), - // If no namespace is specified, all namespaces are watched. - // This is desirable for VolumeSnapshots, as we want to query for all VolumeSnapshots across all namespaces using this informer - snapshotterSharedInformerFactory: snapshotvebeta1informers.NewSharedInformerFactoryWithOptions(csiSnapClient, 0), + namespace: f.Namespace(), + metricsAddress: config.metricsAddress, + kubeClientConfig: clientConfig, + kubeClient: kubeClient, + veleroClient: veleroClient, + discoveryClient: veleroClient.Discovery(), + dynamicClient: dynamicClient, + sharedInformerFactory: informers.NewSharedInformerFactoryWithOptions(veleroClient, 0, informers.WithNamespace(f.Namespace())), + snapshotterSharedInformerFactory: NewCSIInformerFactoryWrapper(csiSnapClient), ctx: ctx, cancelFunc: cancelFunc, logger: logger, @@ -619,8 +618,9 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string // CSI is enabled, and the resources were found. // Instantiate the listers fully s.logger.Debug("Creating CSI listers") - vsLister = s.snapshotterSharedInformerFactory.Snapshot().V1beta1().VolumeSnapshots().Lister() - vscLister = s.snapshotterSharedInformerFactory.Snapshot().V1beta1().VolumeSnapshotContents().Lister() + // Access the wrapped factory directly here since we've already done the feature flag check above to know it's safe. + vsLister = s.snapshotterSharedInformerFactory.factory.Snapshot().V1beta1().VolumeSnapshots().Lister() + vscLister = s.snapshotterSharedInformerFactory.factory.Snapshot().V1beta1().VolumeSnapshotContents().Lister() case err != nil: cmd.CheckError(err) } @@ -884,3 +884,34 @@ func (s *server) runProfiler() { s.logger.WithError(errors.WithStack(err)).Error("error running profiler http server") } } + +// CSIInformerFactoryWrapper is a proxy around the CSI SharedInformerFactory that checks the CSI feature flag before performing operations. +type CSIInformerFactoryWrapper struct { + factory snapshotvebeta1informers.SharedInformerFactory +} + +func NewCSIInformerFactoryWrapper(c snapshotvebeta1client.Interface) *CSIInformerFactoryWrapper { + // If no namespace is specified, all namespaces are watched. + // This is desirable for VolumeSnapshots, as we want to query for all VolumeSnapshots across all namespaces using this informer + w := &CSIInformerFactoryWrapper{} + + if features.IsEnabled("EnableCSI") { + w.factory = snapshotvebeta1informers.NewSharedInformerFactoryWithOptions(c, 0) + } + return w +} + +// Start proxies the Start call to the CSI SharedInformerFactory. +func (w *CSIInformerFactoryWrapper) Start(stopCh <-chan struct{}) { + if features.IsEnabled("EnableCSI") { + w.factory.Start(stopCh) + } +} + +// WaitForCacheSync proxies the WaitForCacheSync call to the CSI SharedInformerFactory. +func (w *CSIInformerFactoryWrapper) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + if features.IsEnabled("EnableCSI") { + return w.factory.WaitForCacheSync(stopCh) + } + return make(map[reflect.Type]bool, 0) +} From 9719e4de9df938ffbbd2176eebc91f122911653c Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Tue, 31 Mar 2020 18:06:54 -0400 Subject: [PATCH 10/26] Don't defer cancelFunc, since it causes issues Infomers won't start if cancelFunc is invoked as soon as the newServer function exits via the defer Signed-off-by: Nolan Brubaker --- pkg/cmd/server/server.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index c91f9f6bf..9b06cf29d 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -270,11 +270,15 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s return nil, err } + // cancelFunc is not deferred here because if it was, then ctx would immediately + // be cancelled once this function exited, making it useless to any informers using later. + // That, in turn, causes the velero server to halt when the first informer tries to use it (probably restic's). + // Therefore, we must explicitly call it on the error paths in this function. ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() clientConfig, err := f.ClientConfig() if err != nil { + cancelFunc() return nil, err } @@ -282,6 +286,7 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s if features.IsEnabled("EnableCSI") { csiSnapClient, err = snapshotvebeta1client.NewForConfig(clientConfig) if err != nil { + cancelFunc() return nil, err } } From 52028853496cb8f471bb8ceb942b21d114b26340 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Tue, 31 Mar 2020 18:08:01 -0400 Subject: [PATCH 11/26] Remove unused CSI client variable Signed-off-by: Nolan Brubaker --- pkg/cmd/server/server.go | 1 - pkg/controller/backup_controller.go | 4 ---- 2 files changed, 5 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 9b06cf29d..0f04549fb 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -635,7 +635,6 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.sharedInformerFactory.Velero().V1().Backups(), s.veleroClient.VeleroV1(), s.discoveryHelper, - s.csiSnapClient.SnapshotV1beta1(), backupper, s.logger, s.logLevel, diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index ca7e0aaf7..914793838 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -38,7 +38,6 @@ import ( "k8s.io/client-go/tools/cache" snapshotv1beta1api "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" - snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1" snapshotv1beta1listers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -65,7 +64,6 @@ type backupController struct { backupper pkgbackup.Backupper lister velerov1listers.BackupLister client velerov1client.BackupsGetter - csiClient snapshotv1beta1.SnapshotV1beta1Interface clock clock.Clock backupLogLevel logrus.Level newPluginManager func(logrus.FieldLogger) clientmgmt.Manager @@ -88,7 +86,6 @@ func NewBackupController( backupInformer velerov1informers.BackupInformer, client velerov1client.BackupsGetter, discoveryHelper discovery.Helper, - csiClient snapshotv1beta1.SnapshotV1beta1Interface, backupper pkgbackup.Backupper, logger logrus.FieldLogger, backupLogLevel logrus.Level, @@ -110,7 +107,6 @@ func NewBackupController( backupper: backupper, lister: backupInformer.Lister(), client: client, - csiClient: csiClient, clock: &clock.RealClock{}, backupLogLevel: backupLogLevel, newPluginManager: newPluginManager, From 8c11afa30ae24e8bae30081c4cf0261a2f4fc181 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Fri, 10 Apr 2020 12:52:42 -0400 Subject: [PATCH 12/26] Update dependencies to latest versio Signed-off-by: Nolan Brubaker --- go.mod | 1 - go.sum | 25 +++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a90d0de93..a91c7cd7a 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,6 @@ require ( github.com/gobwas/glob v0.2.3 github.com/gofrs/uuid v3.2.0+incompatible github.com/golang/protobuf v1.3.2 - github.com/google/go-cmp v0.3.1 // indirect github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect diff --git a/go.sum b/go.sum index 397a13641..d26c1bab7 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,7 @@ github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 h1:llBx5m8Gk0lrA github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -95,6 +96,7 @@ github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -157,8 +159,10 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -182,6 +186,7 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -189,6 +194,7 @@ github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsC github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -206,10 +212,12 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -222,16 +230,20 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= @@ -272,8 +284,10 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -282,6 +296,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -306,10 +321,13 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -324,10 +342,12 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -472,6 +492,7 @@ google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -493,15 +514,18 @@ google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -540,6 +564,7 @@ k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUc k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a h1:UcxjrRMyNx/i/y8G7kPvLyy7rfbeuf1PYyBf973pgyU= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kubernetes v1.14.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= From 9c1a2e884b731b1578bbbdd1f295b67cf570eb57 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Fri, 10 Apr 2020 12:58:58 -0400 Subject: [PATCH 13/26] Update comments Signed-off-by: Nolan Brubaker --- pkg/controller/backup_controller.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 914793838..02d5ff52e 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -559,6 +559,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { //TODO: Is this right? backupLog.Error(err) } + // Currently, VSCs are not being labelled in the plugin, so none will be returned here. volumeSnapshotContents, err = c.volumeSnapshotContentLister.List(selector) if err != nil { //TODO: Is this right? @@ -638,7 +639,7 @@ func persistBackup(backup *pkgbackup.Request, csiVolumeSnapshots []*snapshotv1beta1api.VolumeSnapshot, volumeSnapshotContents []*snapshotv1beta1api.VolumeSnapshotContent, ) []error { - // TODO(nrb-csi): Reduce duplication in this function + // TODO(nrb-csi): Reduce duplication in this function once uploads actually work errs := []error{} backupJSON := new(bytes.Buffer) From 850554911f52adc1d303b0cdfa425ee23244c8d9 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Mon, 13 Apr 2020 13:37:39 -0400 Subject: [PATCH 14/26] Reduce duplication in object storage uploads Signed-off-by: Nolan Brubaker --- pkg/persistence/object_store.go | 65 ++++++--------------------------- 1 file changed, 12 insertions(+), 53 deletions(-) diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 2b4169553..2e9285ab3 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -218,63 +218,22 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error { return kerrors.NewAggregate([]error{err, deleteErr}) } - // TODO: How do we reduce duplication from here to the end of the function? - if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getPodVolumeBackupsKey(info.Name), info.PodVolumeBackups); err != nil { - errs := []error{err} - - deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getBackupContentsKey(info.Name)) - errs = append(errs, deleteErr) - - deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) - errs = append(errs, deleteErr) - - return kerrors.NewAggregate(errs) + // Since the logic for all of these files is the exact same except for the name and the contents, + // use a map literal to iterate through them and write them to the bucket. + var backupObjs = map[string]io.Reader{ + s.layout.getPodVolumeBackupsKey(info.Name): info.PodVolumeBackups, + s.layout.getBackupVolumeSnapshotsKey(info.Name): info.VolumeSnapshots, + s.layout.getBackupResourceListKey(info.Name): info.BackupResourceList, + s.layout.getCSIVolumeSnapshotKey(info.Name): info.CSIVolumeSnapshots, + s.layout.getVolumeSnapshotContentsKey(info.Name): info.VolumeSnapshotContents, } - if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupVolumeSnapshotsKey(info.Name), info.VolumeSnapshots); err != nil { - errs := []error{err} - - deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getBackupContentsKey(info.Name)) - errs = append(errs, deleteErr) - - deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) - errs = append(errs, deleteErr) - - return kerrors.NewAggregate(errs) - } - - if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getBackupResourceListKey(info.Name), info.BackupResourceList); err != nil { - errs := []error{err} - - deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getBackupContentsKey(info.Name)) - errs = append(errs, deleteErr) - - deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) - errs = append(errs, deleteErr) - - return kerrors.NewAggregate(errs) - } - - // TODO(nrb-csi): Add tests? - if info.CSIVolumeSnapshots != nil { - if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getCSIVolumeSnapshotKey(info.Name), info.CSIVolumeSnapshots); err != nil { + for key, reader := range backupObjs { + if err := seekAndPutObject(s.objectStore, s.bucket, key, reader); err != nil { errs := []error{err} - deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getCSIVolumeSnapshotKey(info.Name)) - errs = append(errs, deleteErr) - - deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) - errs = append(errs, deleteErr) - - return kerrors.NewAggregate(errs) - } - } - - if info.VolumeSnapshotContents != nil { - if err := seekAndPutObject(s.objectStore, s.bucket, s.layout.getVolumeSnapshotContentsKey(info.Name), info.VolumeSnapshotContents); err != nil { - errs := []error{err} - - deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getVolumeSnapshotContentsKey(info.Name)) + // attempt to clean up the object in case it was made but we couldn't write contents. + deleteErr := s.objectStore.DeleteObject(s.bucket, key) errs = append(errs, deleteErr) deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) From 9372eaa31b0565768d1bfe107206504e8433464d Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Mon, 13 Apr 2020 13:38:11 -0400 Subject: [PATCH 15/26] Reduce duplication in object serialization Signed-off-by: Nolan Brubaker --- pkg/controller/backup_controller.go | 73 ++++++++++++----------------- 1 file changed, 31 insertions(+), 42 deletions(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 02d5ff52e..fc2cc46c2 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -639,7 +639,6 @@ func persistBackup(backup *pkgbackup.Request, csiVolumeSnapshots []*snapshotv1beta1api.VolumeSnapshot, volumeSnapshotContents []*snapshotv1beta1api.VolumeSnapshotContent, ) []error { - // TODO(nrb-csi): Reduce duplication in this function once uploads actually work errs := []error{} backupJSON := new(bytes.Buffer) @@ -647,55 +646,30 @@ func persistBackup(backup *pkgbackup.Request, errs = append(errs, errors.Wrap(err, "error encoding backup")) } - // Velero-native volume snapshots - nativeVolumeSnapshots := new(bytes.Buffer) - gzw := gzip.NewWriter(nativeVolumeSnapshots) - - if err := json.NewEncoder(gzw).Encode(backup.VolumeSnapshots); err != nil { - errs = append(errs, errors.Wrap(err, "error encoding list of velero volume snapshots")) - } - if err := gzw.Close(); err != nil { - errs = append(errs, errors.Wrap(err, "error closing gzip writer")) + // Velero-native volume snapshots (as opposed to CSI ones) + nativeVolumeSnapshots, err := encodeToJSONGzip(backup.VolumeSnapshots) + if err != nil { + errs = append(errs, err) } - podVolumeBackups := new(bytes.Buffer) - gzw = gzip.NewWriter(podVolumeBackups) - - if err := json.NewEncoder(gzw).Encode(backup.PodVolumeBackups); err != nil { - errs = append(errs, errors.Wrap(err, "error encoding pod volume backups")) - } - if err := gzw.Close(); err != nil { - errs = append(errs, errors.Wrap(err, "error closing gzip writer")) + podVolumeBackups, err := encodeToJSONGzip(backup.PodVolumeBackups) + if err != nil { + errs = append(errs, err) } - csiSnapshotJSON := new(bytes.Buffer) - gzw = gzip.NewWriter(csiSnapshotJSON) - - if err := json.NewEncoder(gzw).Encode(csiVolumeSnapshots); err != nil { - errs = append(errs, errors.Wrap(err, "error encoding list of csi volume snapshots")) - } - if err := gzw.Close(); err != nil { - errs = append(errs, errors.Wrap(err, "error closing gzip writer")) + csiSnapshotJSON, err := encodeToJSONGzip(csiVolumeSnapshots) + if err != nil { + errs = append(errs, err) } - snapshotContentsJSON := new(bytes.Buffer) - gzw = gzip.NewWriter(snapshotContentsJSON) - - if err := json.NewEncoder(gzw).Encode(volumeSnapshotContents); err != nil { - errs = append(errs, errors.Wrap(err, "error encoding list of csi volume snapshots")) - } - if err := gzw.Close(); err != nil { - errs = append(errs, errors.Wrap(err, "error closing gzip writer")) + snapshotContentsJSON, err := encodeToJSONGzip(volumeSnapshotContents) + if err != nil { + errs = append(errs, err) } - backupResourceList := new(bytes.Buffer) - gzw = gzip.NewWriter(backupResourceList) - - if err := json.NewEncoder(gzw).Encode(backup.BackupResourceList()); err != nil { - errs = append(errs, errors.Wrap(err, "error encoding backup resource list")) - } - if err := gzw.Close(); err != nil { - errs = append(errs, errors.Wrap(err, "error closing gzip writer")) + backupResourceList, err := encodeToJSONGzip(backup.BackupResourceList()) + if err != nil { + errs = append(errs, err) } if len(errs) > 0 { @@ -734,3 +708,18 @@ func closeAndRemoveFile(file *os.File, log logrus.FieldLogger) { log.WithError(err).WithField("file", file.Name()).Error("error removing file") } } + +// encodeToJSONGzip takes arbitrary Go data and encodes it to GZip compressed JSON in a buffer. +func encodeToJSONGzip(data interface{}) (*bytes.Buffer, error) { + buf := new(bytes.Buffer) + gzw := gzip.NewWriter(buf) + + if err := json.NewEncoder(gzw).Encode(data); err != nil { + return nil, errors.Wrap(err, "error encoding data") + } + if err := gzw.Close(); err != nil { + return nil, errors.Wrap(err, "error closing gzip writer") + } + + return buf, nil +} From 6c3fc8c7801b581deeecd2603c866f8fdf6f0c33 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 16 Apr 2020 14:07:29 -0400 Subject: [PATCH 16/26] Addresss simple review feedback Signed-off-by: Nolan Brubaker --- pkg/cmd/server/server.go | 15 +++++++-------- pkg/controller/backup_controller.go | 2 -- pkg/persistence/object_store_layout.go | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 0f04549fb..ac80f5ae4 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -45,8 +45,8 @@ import ( "k8s.io/client-go/tools/cache" snapshotv1beta1api "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" - snapshotvebeta1client "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" - snapshotvebeta1informers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions" + snapshotv1beta1client "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + snapshotv1beta1informers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions" snapshotv1beta1listers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -223,7 +223,6 @@ type server struct { kubeClientConfig *rest.Config kubeClient kubernetes.Interface veleroClient clientset.Interface - csiSnapClient snapshotvebeta1client.Interface discoveryClient discovery.DiscoveryInterface discoveryHelper velerodiscovery.Helper dynamicClient dynamic.Interface @@ -282,9 +281,9 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s return nil, err } - var csiSnapClient *snapshotvebeta1client.Clientset + var csiSnapClient *snapshotv1beta1client.Clientset if features.IsEnabled("EnableCSI") { - csiSnapClient, err = snapshotvebeta1client.NewForConfig(clientConfig) + csiSnapClient, err = snapshotv1beta1client.NewForConfig(clientConfig) if err != nil { cancelFunc() return nil, err @@ -891,16 +890,16 @@ func (s *server) runProfiler() { // CSIInformerFactoryWrapper is a proxy around the CSI SharedInformerFactory that checks the CSI feature flag before performing operations. type CSIInformerFactoryWrapper struct { - factory snapshotvebeta1informers.SharedInformerFactory + factory snapshotv1beta1informers.SharedInformerFactory } -func NewCSIInformerFactoryWrapper(c snapshotvebeta1client.Interface) *CSIInformerFactoryWrapper { +func NewCSIInformerFactoryWrapper(c snapshotv1beta1client.Interface) *CSIInformerFactoryWrapper { // If no namespace is specified, all namespaces are watched. // This is desirable for VolumeSnapshots, as we want to query for all VolumeSnapshots across all namespaces using this informer w := &CSIInformerFactoryWrapper{} if features.IsEnabled("EnableCSI") { - w.factory = snapshotvebeta1informers.NewSharedInformerFactoryWithOptions(c, 0) + w.factory = snapshotv1beta1informers.NewSharedInformerFactoryWithOptions(c, 0) } return w } diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index fc2cc46c2..5e8d1be7c 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -80,8 +80,6 @@ type backupController struct { volumeSnapshotContentLister snapshotv1beta1listers.VolumeSnapshotContentLister } -// TODO(nrb-csi): Add clients for the VS/VSContent here. -// How about creating them in the server iff EnableCSI, and if they're nil in the backup controller, don't worry about it? func NewBackupController( backupInformer velerov1informers.BackupInformer, client velerov1client.BackupsGetter, diff --git a/pkg/persistence/object_store_layout.go b/pkg/persistence/object_store_layout.go index 70e955968..9accd8e31 100644 --- a/pkg/persistence/object_store_layout.go +++ b/pkg/persistence/object_store_layout.go @@ -101,9 +101,9 @@ func (l *ObjectStoreLayout) getRestoreResultsKey(restore string) string { } func (l *ObjectStoreLayout) getCSIVolumeSnapshotKey(backup string) string { - return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csivolumesnapshot.json.gz", backup)) + return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csi-volumesnapshots.json.gz", backup)) } func (l *ObjectStoreLayout) getVolumeSnapshotContentsKey(backup string) string { - return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-volumesnapshotcontents.json.gz", backup)) + return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csi-volumesnapshotcontents.json.gz", backup)) } From ad18318bb1546732287066cac4f3e47396da8a46 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 16 Apr 2020 14:31:20 -0400 Subject: [PATCH 17/26] Rename snapshotter factory variable, return nil in wrapper Signed-off-by: Nolan Brubaker --- pkg/cmd/server/server.go | 76 ++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index ac80f5ae4..01672dbb0 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -218,24 +218,24 @@ func NewCommand(f client.Factory) *cobra.Command { } type server struct { - namespace string - metricsAddress string - kubeClientConfig *rest.Config - kubeClient kubernetes.Interface - veleroClient clientset.Interface - discoveryClient discovery.DiscoveryInterface - discoveryHelper velerodiscovery.Helper - dynamicClient dynamic.Interface - sharedInformerFactory informers.SharedInformerFactory - snapshotterSharedInformerFactory *CSIInformerFactoryWrapper - ctx context.Context - cancelFunc context.CancelFunc - logger logrus.FieldLogger - logLevel logrus.Level - pluginRegistry clientmgmt.Registry - resticManager restic.RepositoryManager - metrics *metrics.ServerMetrics - config serverConfig + namespace string + metricsAddress string + kubeClientConfig *rest.Config + kubeClient kubernetes.Interface + veleroClient clientset.Interface + discoveryClient discovery.DiscoveryInterface + discoveryHelper velerodiscovery.Helper + dynamicClient dynamic.Interface + sharedInformerFactory informers.SharedInformerFactory + csiSnapshotterSharedInformerFactory *CSIInformerFactoryWrapper + ctx context.Context + cancelFunc context.CancelFunc + logger logrus.FieldLogger + logLevel logrus.Level + pluginRegistry clientmgmt.Registry + resticManager restic.RepositoryManager + metrics *metrics.ServerMetrics + config serverConfig } func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*server, error) { @@ -291,21 +291,21 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s } s := &server{ - namespace: f.Namespace(), - metricsAddress: config.metricsAddress, - kubeClientConfig: clientConfig, - kubeClient: kubeClient, - veleroClient: veleroClient, - discoveryClient: veleroClient.Discovery(), - dynamicClient: dynamicClient, - sharedInformerFactory: informers.NewSharedInformerFactoryWithOptions(veleroClient, 0, informers.WithNamespace(f.Namespace())), - snapshotterSharedInformerFactory: NewCSIInformerFactoryWrapper(csiSnapClient), - ctx: ctx, - cancelFunc: cancelFunc, - logger: logger, - logLevel: logger.Level, - pluginRegistry: pluginRegistry, - config: config, + namespace: f.Namespace(), + metricsAddress: config.metricsAddress, + kubeClientConfig: clientConfig, + kubeClient: kubeClient, + veleroClient: veleroClient, + discoveryClient: veleroClient.Discovery(), + dynamicClient: dynamicClient, + sharedInformerFactory: informers.NewSharedInformerFactoryWithOptions(veleroClient, 0, informers.WithNamespace(f.Namespace())), + csiSnapshotterSharedInformerFactory: NewCSIInformerFactoryWrapper(csiSnapClient), + ctx: ctx, + cancelFunc: cancelFunc, + logger: logger, + logLevel: logger.Level, + pluginRegistry: pluginRegistry, + config: config, } return s, nil @@ -623,8 +623,8 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string // Instantiate the listers fully s.logger.Debug("Creating CSI listers") // Access the wrapped factory directly here since we've already done the feature flag check above to know it's safe. - vsLister = s.snapshotterSharedInformerFactory.factory.Snapshot().V1beta1().VolumeSnapshots().Lister() - vscLister = s.snapshotterSharedInformerFactory.factory.Snapshot().V1beta1().VolumeSnapshotContents().Lister() + vsLister = s.csiSnapshotterSharedInformerFactory.factory.Snapshot().V1beta1().VolumeSnapshots().Lister() + vscLister = s.csiSnapshotterSharedInformerFactory.factory.Snapshot().V1beta1().VolumeSnapshotContents().Lister() case err != nil: cmd.CheckError(err) } @@ -836,10 +836,10 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string // start the informers & and wait for the caches to sync s.sharedInformerFactory.Start(ctx.Done()) - s.snapshotterSharedInformerFactory.Start(ctx.Done()) + s.csiSnapshotterSharedInformerFactory.Start(ctx.Done()) s.logger.Info("Waiting for informer caches to sync") cacheSyncResults := s.sharedInformerFactory.WaitForCacheSync(ctx.Done()) - csiCacheSyncResults := s.snapshotterSharedInformerFactory.WaitForCacheSync(ctx.Done()) + csiCacheSyncResults := s.csiSnapshotterSharedInformerFactory.WaitForCacheSync(ctx.Done()) s.logger.Info("Done waiting for informer caches to sync") // Append our CSI informer types into the larger list of caches, so we can check them all at once @@ -916,5 +916,5 @@ func (w *CSIInformerFactoryWrapper) WaitForCacheSync(stopCh <-chan struct{}) map if features.IsEnabled("EnableCSI") { return w.factory.WaitForCacheSync(stopCh) } - return make(map[reflect.Type]bool, 0) + return nil } From b4e18b489c55a80b3c0460c9074b72307bef81e1 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 16 Apr 2020 15:17:41 -0400 Subject: [PATCH 18/26] Clean up object contents when other files fail Signed-off-by: Nolan Brubaker --- pkg/persistence/object_store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 2e9285ab3..16db4796c 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -232,8 +232,8 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error { if err := seekAndPutObject(s.objectStore, s.bucket, key, reader); err != nil { errs := []error{err} - // attempt to clean up the object in case it was made but we couldn't write contents. - deleteErr := s.objectStore.DeleteObject(s.bucket, key) + // attempt to clean up the backup contents and metadata if we fail to upload and of the extra files. + deleteErr := s.objectStore.DeleteObject(s.bucket, s.layout.getBackupContentsKey(info.Name)) errs = append(errs, deleteErr) deleteErr = s.objectStore.DeleteObject(s.bucket, s.layout.getBackupMetadataKey(info.Name)) From 15d4c11305d406c48e5c6d5085b5e86b060eab95 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 16 Apr 2020 15:28:29 -0400 Subject: [PATCH 19/26] Wrap CSI lister access in a nil check Signed-off-by: Nolan Brubaker --- pkg/controller/backup_controller.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 5e8d1be7c..7a25d87ea 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -551,18 +551,22 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { if features.IsEnabled("EnableCSI") { selector := labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: backup.Name}) - // TODO(nrb-csi): Only run listers methods if the listers aren't nil, just in case - volumeSnapshots, err = c.volumeSnapshotLister.List(selector) - if err != nil { - //TODO: Is this right? - backupLog.Error(err) + // Listers are wrapped in a nil check out of caution, since they may not be populated based on the + // EnableCSI feature flag. This is more to guard against programmer error, as they shouldn't be nil + // when EnableCSI is on. + if c.volumeSnapshotLister != nil { + volumeSnapshots, err = c.volumeSnapshotLister.List(selector) + if err != nil { + backupLog.Error(err) + } } - // Currently, VSCs are not being labelled in the plugin, so none will be returned here. - volumeSnapshotContents, err = c.volumeSnapshotContentLister.List(selector) - if err != nil { - //TODO: Is this right? - backupLog.Error(err) + if c.volumeSnapshotContentLister != nil { + volumeSnapshotContents, err = c.volumeSnapshotContentLister.List(selector) + if err != nil { + backupLog.Error(err) + + } } } From d31951c81c17c9f9c212f5ca9d8576618379738a Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 16 Apr 2020 15:36:43 -0400 Subject: [PATCH 20/26] Add CSI feature flag constant Signed-off-by: Nolan Brubaker --- pkg/apis/velero/v1/constants.go | 3 +++ pkg/cmd/server/server.go | 12 ++++++------ pkg/controller/backup_controller.go | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/apis/velero/v1/constants.go b/pkg/apis/velero/v1/constants.go index 72a084ea4..2d216858f 100644 --- a/pkg/apis/velero/v1/constants.go +++ b/pkg/apis/velero/v1/constants.go @@ -36,4 +36,7 @@ const ( // NamespaceScopedDir is the name of the directory containing namespace-scoped // resource within a Velero backup. NamespaceScopedDir = "namespaces" + + // CSIFeatureFlag is the feature flag string that defines whether or not CSI features are being used. + CSIFeatureFlag = "EnableCSI" ) diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 01672dbb0..76810af93 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -282,7 +282,7 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s } var csiSnapClient *snapshotv1beta1client.Clientset - if features.IsEnabled("EnableCSI") { + if features.IsEnabled(api.CSIFeatureFlag) { csiSnapClient, err = snapshotv1beta1client.NewForConfig(clientConfig) if err != nil { cancelFunc() @@ -612,12 +612,12 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string // If CSI is enabled, check for the CSI groups and generate the listers // If CSI isn't enabled, proceed normally. - if features.IsEnabled("EnableCSI") { + if features.IsEnabled(api.CSIFeatureFlag) { _, err = s.discoveryClient.ServerResourcesForGroupVersion(snapshotv1beta1api.SchemeGroupVersion.String()) switch { case apierrors.IsNotFound(err): // CSI is enabled, but the required CRDs aren't installed, so halt. - s.logger.Fatalf("The 'EnableCSI' feature flag was specified, but CSI API group [%s] was not found.", snapshotv1beta1api.SchemeGroupVersion.String()) + s.logger.Fatalf("The '%s' feature flag was specified, but CSI API group [%s] was not found.", api.CSIFeatureFlag, snapshotv1beta1api.SchemeGroupVersion.String()) case err == nil: // CSI is enabled, and the resources were found. // Instantiate the listers fully @@ -898,7 +898,7 @@ func NewCSIInformerFactoryWrapper(c snapshotv1beta1client.Interface) *CSIInforme // This is desirable for VolumeSnapshots, as we want to query for all VolumeSnapshots across all namespaces using this informer w := &CSIInformerFactoryWrapper{} - if features.IsEnabled("EnableCSI") { + if features.IsEnabled(api.CSIFeatureFlag) { w.factory = snapshotv1beta1informers.NewSharedInformerFactoryWithOptions(c, 0) } return w @@ -906,14 +906,14 @@ func NewCSIInformerFactoryWrapper(c snapshotv1beta1client.Interface) *CSIInforme // Start proxies the Start call to the CSI SharedInformerFactory. func (w *CSIInformerFactoryWrapper) Start(stopCh <-chan struct{}) { - if features.IsEnabled("EnableCSI") { + if features.IsEnabled(api.CSIFeatureFlag) { w.factory.Start(stopCh) } } // WaitForCacheSync proxies the WaitForCacheSync call to the CSI SharedInformerFactory. func (w *CSIInformerFactoryWrapper) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { - if features.IsEnabled("EnableCSI") { + if features.IsEnabled(api.CSIFeatureFlag) { return w.factory.WaitForCacheSync(stopCh) } return nil diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 7a25d87ea..78fc84f75 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -548,7 +548,7 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { // This way, we only make the Lister call if the feature flag's on. var volumeSnapshots []*snapshotv1beta1api.VolumeSnapshot var volumeSnapshotContents []*snapshotv1beta1api.VolumeSnapshotContent - if features.IsEnabled("EnableCSI") { + if features.IsEnabled(velerov1api.CSIFeatureFlag) { selector := labels.SelectorFromSet(map[string]string{velerov1api.BackupNameLabel: backup.Name}) // Listers are wrapped in a nil check out of caution, since they may not be populated based on the From b567859655fd047a4f5e192ea1f4f0f036de297b Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Thu, 16 Apr 2020 15:42:57 -0400 Subject: [PATCH 21/26] Add data description for encoding JSON in case of error Signed-off-by: Nolan Brubaker --- pkg/controller/backup_controller.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 78fc84f75..f57d80f23 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -649,27 +649,27 @@ func persistBackup(backup *pkgbackup.Request, } // Velero-native volume snapshots (as opposed to CSI ones) - nativeVolumeSnapshots, err := encodeToJSONGzip(backup.VolumeSnapshots) + nativeVolumeSnapshots, err := encodeToJSONGzip(backup.VolumeSnapshots, "native volumesnapshots list") if err != nil { errs = append(errs, err) } - podVolumeBackups, err := encodeToJSONGzip(backup.PodVolumeBackups) + podVolumeBackups, err := encodeToJSONGzip(backup.PodVolumeBackups, "pod volume backups list") if err != nil { errs = append(errs, err) } - csiSnapshotJSON, err := encodeToJSONGzip(csiVolumeSnapshots) + csiSnapshotJSON, err := encodeToJSONGzip(csiVolumeSnapshots, "csi volume snapshots list") if err != nil { errs = append(errs, err) } - snapshotContentsJSON, err := encodeToJSONGzip(volumeSnapshotContents) + snapshotContentsJSON, err := encodeToJSONGzip(volumeSnapshotContents, "volume snapshot contents list") if err != nil { errs = append(errs, err) } - backupResourceList, err := encodeToJSONGzip(backup.BackupResourceList()) + backupResourceList, err := encodeToJSONGzip(backup.BackupResourceList(), "backup resources list") if err != nil { errs = append(errs, err) } @@ -711,13 +711,13 @@ func closeAndRemoveFile(file *os.File, log logrus.FieldLogger) { } } -// encodeToJSONGzip takes arbitrary Go data and encodes it to GZip compressed JSON in a buffer. -func encodeToJSONGzip(data interface{}) (*bytes.Buffer, error) { +// encodeToJSONGzip takes arbitrary Go data and encodes it to GZip compressed JSON in a buffer, as well as a description of the data to put into an error should encoding fail. +func encodeToJSONGzip(data interface{}, desc string) (*bytes.Buffer, error) { buf := new(bytes.Buffer) gzw := gzip.NewWriter(buf) if err := json.NewEncoder(gzw).Encode(data); err != nil { - return nil, errors.Wrap(err, "error encoding data") + return nil, errors.Wrapf(err, "error encoding %s", desc) } if err := gzw.Close(); err != nil { return nil, errors.Wrap(err, "error closing gzip writer") From eefd12b3e48323ec59f88ef5bbbf8251fad04a26 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Fri, 17 Apr 2020 16:05:29 -0400 Subject: [PATCH 22/26] Rename CSI variables to be more descriptive Signed-off-by: Nolan Brubaker --- pkg/controller/backup_controller.go | 24 ++++++++++++------------ pkg/persistence/object_store.go | 12 ++++++------ pkg/persistence/object_store_layout.go | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index f57d80f23..39b7b30ab 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -639,7 +639,7 @@ func persistBackup(backup *pkgbackup.Request, backupStore persistence.BackupStore, log logrus.FieldLogger, csiVolumeSnapshots []*snapshotv1beta1api.VolumeSnapshot, - volumeSnapshotContents []*snapshotv1beta1api.VolumeSnapshotContent, + csiVolumeSnapshotContents []*snapshotv1beta1api.VolumeSnapshotContent, ) []error { errs := []error{} backupJSON := new(bytes.Buffer) @@ -664,7 +664,7 @@ func persistBackup(backup *pkgbackup.Request, errs = append(errs, err) } - snapshotContentsJSON, err := encodeToJSONGzip(volumeSnapshotContents, "volume snapshot contents list") + csiSnapshotContentsJSON, err := encodeToJSONGzip(csiVolumeSnapshotContents, "csi volume snapshot contents list") if err != nil { errs = append(errs, err) } @@ -681,19 +681,19 @@ func persistBackup(backup *pkgbackup.Request, nativeVolumeSnapshots = nil backupResourceList = nil csiSnapshotJSON = nil - snapshotContentsJSON = nil + csiSnapshotContentsJSON = nil } backupInfo := persistence.BackupInfo{ - Name: backup.Name, - Metadata: backupJSON, - Contents: backupContents, - Log: backupLog, - PodVolumeBackups: podVolumeBackups, - VolumeSnapshots: nativeVolumeSnapshots, - BackupResourceList: backupResourceList, - CSIVolumeSnapshots: csiSnapshotJSON, - VolumeSnapshotContents: snapshotContentsJSON, + Name: backup.Name, + Metadata: backupJSON, + Contents: backupContents, + Log: backupLog, + PodVolumeBackups: podVolumeBackups, + VolumeSnapshots: nativeVolumeSnapshots, + BackupResourceList: backupResourceList, + CSIVolumeSnapshots: csiSnapshotJSON, + CSIVolumeSnapshotContents: csiSnapshotContentsJSON, } if err := backupStore.PutBackup(backupInfo); err != nil { errs = append(errs, err) diff --git a/pkg/persistence/object_store.go b/pkg/persistence/object_store.go index 16db4796c..d15bc8a21 100644 --- a/pkg/persistence/object_store.go +++ b/pkg/persistence/object_store.go @@ -43,7 +43,7 @@ type BackupInfo struct { VolumeSnapshots, BackupResourceList, CSIVolumeSnapshots, - VolumeSnapshotContents io.Reader + CSIVolumeSnapshotContents io.Reader } // BackupStore defines operations for creating, retrieving, and deleting @@ -221,11 +221,11 @@ func (s *objectBackupStore) PutBackup(info BackupInfo) error { // Since the logic for all of these files is the exact same except for the name and the contents, // use a map literal to iterate through them and write them to the bucket. var backupObjs = map[string]io.Reader{ - s.layout.getPodVolumeBackupsKey(info.Name): info.PodVolumeBackups, - s.layout.getBackupVolumeSnapshotsKey(info.Name): info.VolumeSnapshots, - s.layout.getBackupResourceListKey(info.Name): info.BackupResourceList, - s.layout.getCSIVolumeSnapshotKey(info.Name): info.CSIVolumeSnapshots, - s.layout.getVolumeSnapshotContentsKey(info.Name): info.VolumeSnapshotContents, + s.layout.getPodVolumeBackupsKey(info.Name): info.PodVolumeBackups, + s.layout.getBackupVolumeSnapshotsKey(info.Name): info.VolumeSnapshots, + s.layout.getBackupResourceListKey(info.Name): info.BackupResourceList, + s.layout.getCSIVolumeSnapshotKey(info.Name): info.CSIVolumeSnapshots, + s.layout.getCSIVolumeSnapshotContentsKey(info.Name): info.CSIVolumeSnapshotContents, } for key, reader := range backupObjs { diff --git a/pkg/persistence/object_store_layout.go b/pkg/persistence/object_store_layout.go index 9accd8e31..025da966b 100644 --- a/pkg/persistence/object_store_layout.go +++ b/pkg/persistence/object_store_layout.go @@ -104,6 +104,6 @@ func (l *ObjectStoreLayout) getCSIVolumeSnapshotKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csi-volumesnapshots.json.gz", backup)) } -func (l *ObjectStoreLayout) getVolumeSnapshotContentsKey(backup string) string { +func (l *ObjectStoreLayout) getCSIVolumeSnapshotContentsKey(backup string) string { return path.Join(l.subdirs["backups"], backup, fmt.Sprintf("%s-csi-volumesnapshotcontents.json.gz", backup)) } From bc04c568cde47709cbec86f1d02ac29554632cbe Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Fri, 17 Apr 2020 18:26:41 -0400 Subject: [PATCH 23/26] Return list of errors for encoding/zipping Signed-off-by: Nolan Brubaker --- pkg/controller/backup_controller.go | 56 ++++++++++++++++------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 39b7b30ab..24787198d 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -641,40 +641,40 @@ func persistBackup(backup *pkgbackup.Request, csiVolumeSnapshots []*snapshotv1beta1api.VolumeSnapshot, csiVolumeSnapshotContents []*snapshotv1beta1api.VolumeSnapshotContent, ) []error { - errs := []error{} + persistErrs := []error{} backupJSON := new(bytes.Buffer) - if err := encode.EncodeTo(backup.Backup, "json", backupJSON); err != nil { - errs = append(errs, errors.Wrap(err, "error encoding backup")) + if errs := encode.EncodeTo(backup.Backup, "json", backupJSON); errs != nil { + persistErrs = append(persistErrs, errors.Wrap(errs, "error encoding backup")) } // Velero-native volume snapshots (as opposed to CSI ones) - nativeVolumeSnapshots, err := encodeToJSONGzip(backup.VolumeSnapshots, "native volumesnapshots list") - if err != nil { - errs = append(errs, err) + nativeVolumeSnapshots, errs := encodeToJSONGzip(backup.VolumeSnapshots, "native volumesnapshots list") + if errs != nil { + persistErrs = append(persistErrs, errs...) } - podVolumeBackups, err := encodeToJSONGzip(backup.PodVolumeBackups, "pod volume backups list") - if err != nil { - errs = append(errs, err) + podVolumeBackups, errs := encodeToJSONGzip(backup.PodVolumeBackups, "pod volume backups list") + if errs != nil { + persistErrs = append(persistErrs, errs...) } - csiSnapshotJSON, err := encodeToJSONGzip(csiVolumeSnapshots, "csi volume snapshots list") - if err != nil { - errs = append(errs, err) + csiSnapshotJSON, errs := encodeToJSONGzip(csiVolumeSnapshots, "csi volume snapshots list") + if errs != nil { + persistErrs = append(persistErrs, errs...) } - csiSnapshotContentsJSON, err := encodeToJSONGzip(csiVolumeSnapshotContents, "csi volume snapshot contents list") - if err != nil { - errs = append(errs, err) + csiSnapshotContentsJSON, errs := encodeToJSONGzip(csiVolumeSnapshotContents, "csi volume snapshot contents list") + if errs != nil { + persistErrs = append(persistErrs, errs...) } - backupResourceList, err := encodeToJSONGzip(backup.BackupResourceList(), "backup resources list") - if err != nil { - errs = append(errs, err) + backupResourceList, errs := encodeToJSONGzip(backup.BackupResourceList(), "backup resources list") + if errs != nil { + persistErrs = append(persistErrs, errs...) } - if len(errs) > 0 { + if len(persistErrs) > 0 { // Don't upload the JSON files or backup tarball if encoding to json fails. backupJSON = nil backupContents = nil @@ -696,10 +696,10 @@ func persistBackup(backup *pkgbackup.Request, CSIVolumeSnapshotContents: csiSnapshotContentsJSON, } if err := backupStore.PutBackup(backupInfo); err != nil { - errs = append(errs, err) + persistErrs = append(persistErrs, err) } - return errs + return persistErrs } func closeAndRemoveFile(file *os.File, log logrus.FieldLogger) { @@ -712,15 +712,23 @@ func closeAndRemoveFile(file *os.File, log logrus.FieldLogger) { } // encodeToJSONGzip takes arbitrary Go data and encodes it to GZip compressed JSON in a buffer, as well as a description of the data to put into an error should encoding fail. -func encodeToJSONGzip(data interface{}, desc string) (*bytes.Buffer, error) { +func encodeToJSONGzip(data interface{}, desc string) (*bytes.Buffer, []error) { buf := new(bytes.Buffer) gzw := gzip.NewWriter(buf) + // Since both encoding and closing the gzip writer could fail separately and both errors are useful, + // collect both errors to report back. + errs := []error{} + if err := json.NewEncoder(gzw).Encode(data); err != nil { - return nil, errors.Wrapf(err, "error encoding %s", desc) + errs = append(errs, errors.Wrapf(err, "error encoding %s", desc)) } if err := gzw.Close(); err != nil { - return nil, errors.Wrap(err, "error closing gzip writer") + errs = append(errs, errors.Wrapf(err, "error closing gzip writer for %s", desc)) + } + + if len(errs) > 0 { + return nil, errs } return buf, nil From 16cf2780d8049f10768201fb66543e2b1778df3a Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Fri, 17 Apr 2020 18:42:14 -0400 Subject: [PATCH 24/26] Fetch contents based on volumesnapshots found Signed-off-by: Nolan Brubaker --- pkg/controller/backup_controller.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 24787198d..222ae41b3 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -562,10 +562,14 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { } if c.volumeSnapshotContentLister != nil { - volumeSnapshotContents, err = c.volumeSnapshotContentLister.List(selector) - if err != nil { - backupLog.Error(err) - + // Since VolumeSnapshotContent objects are not currently labeled, get them by using binding from the VolumeSnapshot + for _, vs := range volumeSnapshots { + vsc, err := c.volumeSnapshotContentLister.Get(*vs.Status.BoundVolumeSnapshotContentName) + if err != nil { + backupLog.Error(err) + continue + } + volumeSnapshotContents = append(volumeSnapshotContents, vsc) } } } From aa3abefdbf221f7354342713f1a4e8401ab85f0f Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Mon, 20 Apr 2020 12:33:21 -0400 Subject: [PATCH 25/26] Address review feedback Signed-off-by: Nolan Brubaker --- pkg/controller/backup_controller.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 222ae41b3..b8a9e0d13 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -564,12 +564,15 @@ func (c *backupController) runBackup(backup *pkgbackup.Request) error { if c.volumeSnapshotContentLister != nil { // Since VolumeSnapshotContent objects are not currently labeled, get them by using binding from the VolumeSnapshot for _, vs := range volumeSnapshots { - vsc, err := c.volumeSnapshotContentLister.Get(*vs.Status.BoundVolumeSnapshotContentName) - if err != nil { - backupLog.Error(err) - continue + // nil check just in case the snapshot and the content object did not get bound before returning from the plugins + if vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + vsc, err := c.volumeSnapshotContentLister.Get(*vs.Status.BoundVolumeSnapshotContentName) + if err != nil { + backupLog.Error(err) + continue + } + volumeSnapshotContents = append(volumeSnapshotContents, vsc) } - volumeSnapshotContents = append(volumeSnapshotContents, vsc) } } } @@ -648,8 +651,8 @@ func persistBackup(backup *pkgbackup.Request, persistErrs := []error{} backupJSON := new(bytes.Buffer) - if errs := encode.EncodeTo(backup.Backup, "json", backupJSON); errs != nil { - persistErrs = append(persistErrs, errors.Wrap(errs, "error encoding backup")) + if err := encode.EncodeTo(backup.Backup, "json", backupJSON); err != nil { + persistErrs = append(persistErrs, errors.Wrap(err, "error encoding backup")) } // Velero-native volume snapshots (as opposed to CSI ones) From 68ee0ecfedc2bc0eab54d7f4116e598a6752a035 Mon Sep 17 00:00:00 2001 From: Nolan Brubaker Date: Mon, 20 Apr 2020 14:39:45 -0400 Subject: [PATCH 26/26] Add changelog Signed-off-by: Nolan Brubaker --- changelogs/unreleased/2323-nrb | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/unreleased/2323-nrb diff --git a/changelogs/unreleased/2323-nrb b/changelogs/unreleased/2323-nrb new file mode 100644 index 000000000..159af2747 --- /dev/null +++ b/changelogs/unreleased/2323-nrb @@ -0,0 +1 @@ +When the EnableCSI feature flag is provided, upload CSI VolumeSnapshots and VolumeSnapshotContents to object storage as gzipped JSON.